From 057108f471385548fcd191abfc0318c6b4a12981 Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Wed, 23 Oct 2024 21:53:47 +0100 Subject: [PATCH 1/8] Refactor wordpress abtract model --- wp_connector/migrations/0001_initial.py | 6 ++- .../migrations/0002_alter_wpauthor_name.py | 18 ------- ..._id_wpcategory_wagtail_page_id_and_more.py | 48 ------------------- wp_connector/models/abstract.py | 10 +++- 4 files changed, 12 insertions(+), 70 deletions(-) delete mode 100644 wp_connector/migrations/0002_alter_wpauthor_name.py delete mode 100644 wp_connector/migrations/0003_wpauthor_wagtail_page_id_wpcategory_wagtail_page_id_and_more.py diff --git a/wp_connector/migrations/0001_initial.py b/wp_connector/migrations/0001_initial.py index a52250a..1a453ce 100644 --- a/wp_connector/migrations/0001_initial.py +++ b/wp_connector/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-10-16 20:28 +# Generated by Django 5.1.2 on 2024-10-23 20:36 import django.db.models.deletion from django.db import migrations, models @@ -32,7 +32,7 @@ class Migration(migrations.Migration): ("wagtail_model", models.JSONField(blank=True, null=True)), ("wp_cleaned_content", models.TextField(blank=True, null=True)), ("wp_block_content", models.JSONField(blank=True, null=True)), - ("name", models.CharField(max_length=255)), + ("name", models.CharField(help_text="Display name", max_length=255)), ("url", models.URLField(blank=True, null=True)), ("description", models.TextField(blank=True, null=True)), ("link", models.URLField()), @@ -140,6 +140,7 @@ class Migration(migrations.Migration): ("wagtail_model", models.JSONField(blank=True, null=True)), ("wp_cleaned_content", models.TextField(blank=True, null=True)), ("wp_block_content", models.JSONField(blank=True, null=True)), + ("wagtail_page_id", models.IntegerField(blank=True, null=True)), ("title", models.CharField(max_length=255)), ("date", models.DateTimeField()), ("date_gmt", models.DateTimeField()), @@ -201,6 +202,7 @@ class Migration(migrations.Migration): ("wagtail_model", models.JSONField(blank=True, null=True)), ("wp_cleaned_content", models.TextField(blank=True, null=True)), ("wp_block_content", models.JSONField(blank=True, null=True)), + ("wagtail_page_id", models.IntegerField(blank=True, null=True)), ("title", models.CharField(max_length=255)), ("date", models.DateTimeField()), ("date_gmt", models.DateTimeField()), diff --git a/wp_connector/migrations/0002_alter_wpauthor_name.py b/wp_connector/migrations/0002_alter_wpauthor_name.py deleted file mode 100644 index 1903781..0000000 --- a/wp_connector/migrations/0002_alter_wpauthor_name.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-18 11:00 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("wp_connector", "0001_initial"), - ] - - operations = [ - migrations.AlterField( - model_name="wpauthor", - name="name", - field=models.CharField(help_text="Display name", max_length=255), - ), - ] diff --git a/wp_connector/migrations/0003_wpauthor_wagtail_page_id_wpcategory_wagtail_page_id_and_more.py b/wp_connector/migrations/0003_wpauthor_wagtail_page_id_wpcategory_wagtail_page_id_and_more.py deleted file mode 100644 index 05c7d6a..0000000 --- a/wp_connector/migrations/0003_wpauthor_wagtail_page_id_wpcategory_wagtail_page_id_and_more.py +++ /dev/null @@ -1,48 +0,0 @@ -# Generated by Django 5.1.2 on 2024-10-18 12:09 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("wp_connector", "0002_alter_wpauthor_name"), - ] - - operations = [ - migrations.AddField( - model_name="wpauthor", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name="wpcategory", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name="wpcomment", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name="wpmedia", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name="wppage", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name="wppost", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - migrations.AddField( - model_name="wptag", - name="wagtail_page_id", - field=models.IntegerField(blank=True, null=True), - ), - ] diff --git a/wp_connector/models/abstract.py b/wp_connector/models/abstract.py index 0342c99..4e198fc 100644 --- a/wp_connector/models/abstract.py +++ b/wp_connector/models/abstract.py @@ -27,7 +27,9 @@ def __init__(self, *args, **kwargs): wagtail_model = models.JSONField(blank=True, null=True) wp_cleaned_content = models.TextField(blank=True, null=True) wp_block_content = models.JSONField(blank=True, null=True) - wagtail_page_id = models.IntegerField(blank=True, null=True) + + # avoid errors with models not exportable in django admin + wagtail_page_id = models.NOT_PROVIDED class Meta: abstract = True @@ -88,7 +90,11 @@ def get_source_url(self): return self.SOURCE_URL.strip("/") -class ExportableMixin: +class ExportableWordpressModel(WordpressModel): + WAGTAIL_PAGE_MODEL = None # e.g. "blog.BlogPage" + WAGTAIL_PAGE_MODEL_PARENT = None # e.g. "blog.BlogIndexPage" + FIELD_MAPPING = {} # e.g. {"title": "title"} {[object_field]: [wagtail_field]} + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not self.WAGTAIL_PAGE_MODEL: From 871f0196915c567d50f62c7cc8a73f1fbfc2e922 Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Wed, 23 Oct 2024 22:32:59 +0100 Subject: [PATCH 2/8] Handle messages for user --- wp_connector/admin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wp_connector/admin.py b/wp_connector/admin.py index e36b097..bd16da5 100644 --- a/wp_connector/admin.py +++ b/wp_connector/admin.py @@ -454,6 +454,7 @@ def update_wagtail_page(self, admin, request, queryset): exporter.post_init_messages["message"], level=exporter.post_init_messages["level"], ) + if exporter.post_init_messages.get("skip", False): continue From 86e658ee5853c96e8f503675a5a22f99ff47982e Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Thu, 24 Oct 2024 21:54:58 +0100 Subject: [PATCH 3/8] Implement categories for the blog pages --- blog/migrations/0007_blogcategory.py | 32 ++++++++++++++++ blog/migrations/0008_blogpagecategory.py | 48 ++++++++++++++++++++++++ blog/models.py | 2 +- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 blog/migrations/0007_blogcategory.py create mode 100644 blog/migrations/0008_blogpagecategory.py diff --git a/blog/migrations/0007_blogcategory.py b/blog/migrations/0007_blogcategory.py new file mode 100644 index 0000000..6b9aadf --- /dev/null +++ b/blog/migrations/0007_blogcategory.py @@ -0,0 +1,32 @@ +# Generated by Django 5.1.2 on 2024-10-24 15:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("blog", "0006_rename_authors_blogpage_author"), + ] + + operations = [ + migrations.CreateModel( + name="BlogCategory", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=255)), + ("slug", models.SlugField(unique=True)), + ], + options={ + "verbose_name_plural": "Categories", + }, + ), + ] diff --git a/blog/migrations/0008_blogpagecategory.py b/blog/migrations/0008_blogpagecategory.py new file mode 100644 index 0000000..6384bee --- /dev/null +++ b/blog/migrations/0008_blogpagecategory.py @@ -0,0 +1,48 @@ +# Generated by Django 5.1.2 on 2024-10-24 15:48 + +import django.db.models.deletion +import modelcluster.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("blog", "0007_blogcategory"), + ] + + operations = [ + migrations.CreateModel( + name="BlogPageCategory", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "category", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="blog_pages", + to="blog.blogcategory", + ), + ), + ( + "page", + modelcluster.fields.ParentalKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="categories", + to="blog.blogpage", + ), + ), + ], + options={ + "verbose_name_plural": "Categories", + }, + ), + ] diff --git a/blog/models.py b/blog/models.py index d5ac788..54cb093 100644 --- a/blog/models.py +++ b/blog/models.py @@ -3,6 +3,7 @@ from modelcluster.contrib.taggit import ClusterTaggableManager from modelcluster.fields import ParentalKey from taggit.models import TaggedItemBase +from wagtail.fields import RichTextField, StreamField from wagtail.admin.panels import ( FieldPanel, InlinePanel, @@ -10,7 +11,6 @@ TabbedInterface, TitleFieldPanel, ) -from wagtail.fields import RichTextField, StreamField from wagtail.models import Page from wagtail.search import index From 5d18de1d3b4528ef7b977289f364ea5f526ad05b Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Thu, 24 Oct 2024 21:55:31 +0100 Subject: [PATCH 4/8] Update the move page function to be reusable --- wp_connector/admin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wp_connector/admin.py b/wp_connector/admin.py index bd16da5..a475f61 100644 --- a/wp_connector/admin.py +++ b/wp_connector/admin.py @@ -437,9 +437,10 @@ def update_wagtail_page(self, admin, request, queryset): for obj in queryset: if not obj.wagtail_page_id: # skip objects that do not have a wagtail_page_id + message_param = obj.title or obj.wp_id self.handle_message_user( request, - f"Page doen't exist for {obj.title} yet. Use the 'Create New Wagtail Pages' action", + f"Page doesn't exist for {message_param} yet. Use the 'Create New Wagtail Pages' action", level="WARNING", ) continue From 7a9c9b6ff79f2e2dc01f8e41e6e793767cb1b05c Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Fri, 25 Oct 2024 16:48:43 +0100 Subject: [PATCH 5/8] Add a streamfield to the StandardPage model --- home/migrations/0005_alter_standardpage_body.py | 4 ++-- home/migrations/0006_alter_standardpage_body.py | 9 +++------ home/models.py | 2 ++ 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/home/migrations/0005_alter_standardpage_body.py b/home/migrations/0005_alter_standardpage_body.py index d6c5522..5144876 100644 --- a/home/migrations/0005_alter_standardpage_body.py +++ b/home/migrations/0005_alter_standardpage_body.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-10-28 14:23 +# Generated by Django 5.1.2 on 2024-10-25 11:06 import wagtail.fields from django.db import migrations @@ -15,7 +15,7 @@ class Migration(migrations.Migration): model_name="standardpage", name="body", field=wagtail.fields.StreamField( - [("paragraph", 0)], + [("paragrpah", 0)], blank=True, block_lookup={0: ("wagtail.blocks.RichTextBlock", (), {})}, ), diff --git a/home/migrations/0006_alter_standardpage_body.py b/home/migrations/0006_alter_standardpage_body.py index 0730293..20d872c 100644 --- a/home/migrations/0006_alter_standardpage_body.py +++ b/home/migrations/0006_alter_standardpage_body.py @@ -1,4 +1,4 @@ -# Generated by Django 5.1.2 on 2024-10-28 15:21 +# Generated by Django 5.1.2 on 2024-10-25 11:50 import wagtail.fields from django.db import migrations @@ -15,12 +15,9 @@ class Migration(migrations.Migration): model_name="standardpage", name="body", field=wagtail.fields.StreamField( - [("paragraph", 1)], + [("paragraph", 0)], blank=True, - block_lookup={ - 0: ("wagtail.blocks.RichTextBlock", (), {}), - 1: ("wagtail.blocks.StructBlock", [[("content", 0)]], {}), - }, + block_lookup={0: ("wagtail.blocks.RichTextBlock", (), {})}, ), ), ] diff --git a/home/models.py b/home/models.py index 35f1549..e5ea241 100644 --- a/home/models.py +++ b/home/models.py @@ -5,6 +5,8 @@ from home.blocks import StreamBlocks from wp_connector.field_panels import WordpressInfoPanel +from .blocks import StreamBlock + class HomePage(Page): pass From e5677c353bc42101cecc19a7a222882f2641fffa Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Fri, 25 Oct 2024 16:50:57 +0100 Subject: [PATCH 6/8] Works by injecting a manual html string --- wp_connector/exporter.py | 38 ++++++++++++++++++++++++++++++++++++- wp_connector/models/page.py | 1 - 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/wp_connector/exporter.py b/wp_connector/exporter.py index 5a66714..8b2643c 100644 --- a/wp_connector/exporter.py +++ b/wp_connector/exporter.py @@ -1,5 +1,8 @@ -from dataclasses import dataclass +import json +from dataclasses import dataclass, field +from typing import List +from bs4 import BeautifulSoup as bs from django.apps import apps from taggit.models import Tag @@ -53,6 +56,7 @@ class Exporter: wagtail_page_model_has_author: bool = False wagtail_page_model_has_tags: bool = False wagtail_page_model_has_categories: bool = False + wagtail_page_model_streamfields: List = field(default_factory=list) field_mapping: dict = None # this takes precedence over the field_mapping @@ -243,3 +247,35 @@ def do_update_wagtail_page(self): "message": f"Updated wagtail page ID:{wagtail_page.id}", "level": "SUCCESS", } + + +@dataclass +class HTML_to_Streamfield: + """ + Convert HTML to Streamfield JSON + """ + + html: str = None + + stream_data: List = field(default_factory=list) + + def __post_init__(self): + if not self.html: + raise ValueError("HTML is required") + self.stream_data = self.parse_html_to_streamfield_json() + print(self.stream_data) + + def parse_html_to_streamfield_json(self): + soup = bs(self.html, "html.parser") + data = [] + for tag in soup.find_all(): + data.append( + { + "type": "paragraph", + "value": tag.text, + } + ) + return data + + def get_stream_data(self): + return json.dumps(self.stream_data) diff --git a/wp_connector/models/page.py b/wp_connector/models/page.py index fbf413b..bcfa376 100644 --- a/wp_connector/models/page.py +++ b/wp_connector/models/page.py @@ -9,7 +9,6 @@ class WPPage(WordpressModel, ExportableMixin, StreamFieldMixin): SOURCE_URL = "/wp-json/wp/v2/pages" WAGTAIL_PAGE_MODEL = "home.StandardPage" WAGTAIL_PAGE_MODEL_PARENT = "home.HomePage" - FIELD_MAPPING = { "title": "title", "content": "body", From caad5a732301afd1424cd861b6e0f6e310c86e34 Mon Sep 17 00:00:00 2001 From: Nick Moreton Date: Fri, 25 Oct 2024 17:10:36 +0100 Subject: [PATCH 7/8] A little tidy up --- wp_connector/exporter.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/wp_connector/exporter.py b/wp_connector/exporter.py index 8b2643c..c6f67cc 100644 --- a/wp_connector/exporter.py +++ b/wp_connector/exporter.py @@ -255,27 +255,21 @@ class HTML_to_Streamfield: Convert HTML to Streamfield JSON """ - html: str = None + html: str = "" stream_data: List = field(default_factory=list) def __post_init__(self): - if not self.html: - raise ValueError("HTML is required") self.stream_data = self.parse_html_to_streamfield_json() print(self.stream_data) def parse_html_to_streamfield_json(self): soup = bs(self.html, "html.parser") - data = [] - for tag in soup.find_all(): - data.append( - { - "type": "paragraph", - "value": tag.text, - } - ) - return data + cache = [] + for tag in soup.find_all(): # top level tags + if len(cache == 0): + cache.append({"type": "paragraph", "value": tag.text}) + return cache def get_stream_data(self): return json.dumps(self.stream_data) From 7891d484d2b092e6c9872eb23c3af3026cd24634 Mon Sep 17 00:00:00 2001 From: nickmoreton Date: Thu, 31 Oct 2024 03:04:41 +0000 Subject: [PATCH 8/8] Run formatting --- blog/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blog/models.py b/blog/models.py index 54cb093..d5ac788 100644 --- a/blog/models.py +++ b/blog/models.py @@ -3,7 +3,6 @@ from modelcluster.contrib.taggit import ClusterTaggableManager from modelcluster.fields import ParentalKey from taggit.models import TaggedItemBase -from wagtail.fields import RichTextField, StreamField from wagtail.admin.panels import ( FieldPanel, InlinePanel, @@ -11,6 +10,7 @@ TabbedInterface, TitleFieldPanel, ) +from wagtail.fields import RichTextField, StreamField from wagtail.models import Page from wagtail.search import index