Add Webhooks support to new REST API (#1418)

pirate · web-flow · commit 29c794925e0a · 2024-05-06T07:17:36.000-07:00
diff --git a/archivebox/__init__.py b/archivebox/__init__.py
@@ -1 +1,7 @@
 __package__ = 'archivebox'
+
+
+# monkey patch django timezone to add back utc (it was removed in Django 5.0)
+import datetime
+from django.utils import timezone
+timezone.utc = datetime.timezone.utc
diff --git a/archivebox/core/admin.py b/archivebox/core/admin.py
@@ -14,6 +14,8 @@
 from django.contrib.auth import get_user_model
 from django import forms
 
+from signal_webhooks.admin import WebhookAdmin, WebhookModel
+
 from ..util import htmldecode, urldecode, ansi_to_html
 
 from core.models import Snapshot, ArchiveResult, Tag
@@ -102,6 +104,7 @@ def add_view(self, request):
 archivebox_admin = ArchiveBoxAdmin()
 archivebox_admin.register(get_user_model())
 archivebox_admin.register(APIToken)
+archivebox_admin.register(WebhookModel, WebhookAdmin)
 archivebox_admin.disable_action('delete_selected')
 
 class ArchiveResultInline(admin.TabularInline):
diff --git a/archivebox/core/settings.py b/archivebox/core/settings.py
@@ -18,6 +18,7 @@
     CUSTOM_TEMPLATES_DIR,
     SQL_INDEX_FILENAME,
     OUTPUT_DIR,
+    ARCHIVE_DIR,
     LOGS_DIR,
     TIMEZONE,
 
@@ -63,6 +64,7 @@
     'core',
     'api',
 
+    'signal_webhooks',
     'django_extensions',
 ]
 
@@ -253,6 +255,29 @@
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
 
+STORAGES = {
+    "default": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage",
+    },
+    "staticfiles": {
+        "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+    },
+    "archive": {
+        "BACKEND": "django.core.files.storage.FileSystemStorage",
+        "OPTIONS": {
+            "base_url": "/archive/",
+            "location": ARCHIVE_DIR,
+        },
+    },
+    # "personas": {
+    #     "BACKEND": "django.core.files.storage.FileSystemStorage",
+    #     "OPTIONS": {
+    #         "base_url": "/personas/",
+    #         "location": PERSONAS_DIR,
+    #     },
+    # },
+}
+
 ################################################################################
 ### Security Settings
 ################################################################################
@@ -379,3 +404,15 @@ def filter(self, record):
         }
     },
 }
+
+
+# Add default webhook configuration to the User model
+SIGNAL_WEBHOOKS = {
+    "HOOKS": {
+        "django.contrib.auth.models.User": ...,
+        "core.models.Snapshot": ...,
+        "core.models.ArchiveResult": ...,
+        "core.models.Tag": ...,
+        "api.models.APIToken": ...,
+    },
+}
diff --git a/archivebox/index/schema.py b/archivebox/index/schema.py
@@ -4,6 +4,7 @@
 
 DO NOT ADD ANY NEW FEATURES TO THIS FILE, NEW CODE GOES HERE: core/models.py
 
+These are the old types we used to use before ArchiveBox v0.4 (before we switched to Django).
 """
 
 __package__ = 'archivebox.index'
diff --git a/archivebox/main.py b/archivebox/main.py
@@ -230,7 +230,7 @@ def version(quiet: bool=False,
         p = platform.uname()
         print(
             'ArchiveBox v{}'.format(get_version(CONFIG)),
-            *((f'COMMIT_HASH={COMMIT_HASH[:7]}',) if COMMIT_HASH else ()),
+            f'COMMIT_HASH={COMMIT_HASH[:7] if COMMIT_HASH else 'unknown'}',
             f'BUILD_TIME={BUILD_TIME}',
         )
         print(
@@ -1356,7 +1356,7 @@ def manage(args: Optional[List[str]]=None, out_dir: Path=OUTPUT_DIR) -> None:
     if (args and "createsuperuser" in args) and (IN_DOCKER and not IS_TTY):
         stderr('[!] Warning: you need to pass -it to use interactive commands in docker', color='lightyellow')
         stderr('    docker run -it archivebox manage {}'.format(' '.join(args or ['...'])), color='lightyellow')
-        stderr()
+        stderr('')
 
     execute_from_command_line([f'{ARCHIVEBOX_BINARY} manage', *(args or ['help'])])
 
diff --git a/archivebox/manage.py b/archivebox/manage.py
@@ -7,7 +7,7 @@
     # versions of ./manage.py commands whenever possible. When that's not possible
     # (e.g. makemigrations), you can comment out this check temporarily
 
-    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv):
+    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'startapp' in sys.argv):
         print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
         print()
         print('    Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')
diff --git a/package-lock.json b/package-lock.json
diff --git a/pyproject.toml b/pyproject.toml
@@ -35,6 +35,7 @@ dependencies = [
     #  - gallery-dl
     #  - scihubdl
     #  - See Github issues for more...
+    "django-signal-webhooks>=0.3.0",
 ]
 
 homepage = "https://github.com/ArchiveBox/ArchiveBox"
diff --git a/requirements.txt b/requirements.txt
@@ -2,26 +2,33 @@
 # Please do not edit it manually.
 
 annotated-types==0.6.0
+anyio==4.3.0
 asgiref==3.8.1
 asttokens==2.4.1
 brotli==1.1.0; implementation_name == "cpython"
 brotlicffi==1.1.0.0; implementation_name != "cpython"
 certifi==2024.2.2
-cffi==1.16.0; implementation_name != "cpython"
+cffi==1.16.0; platform_python_implementation != "PyPy" or implementation_name != "cpython"
 charset-normalizer==3.3.2
 colorama==0.4.6; sys_platform == "win32"
 croniter==2.0.5
+cryptography==42.0.6
 dateparser==1.2.0
 decorator==5.1.1
 django==5.0.4
 django-auth-ldap==4.8.0
 django-extensions==3.2.3
 django-ninja==1.1.0
+django-settings-holder==0.1.2
+django-signal-webhooks==0.3.0
 exceptiongroup==1.2.1; python_version < "3.11"
 executing==2.0.1
 feedparser==6.0.11
+h11==0.14.0
+httpcore==1.0.5
+httpx==0.27.0
 idna==3.7
-ipython==8.23.0
+ipython==8.24.0
 jedi==0.19.1
 matplotlib-inline==0.1.7
 mutagen==1.47.0
@@ -33,20 +40,21 @@ ptyprocess==0.7.0; sys_platform != "win32" and sys_platform != "emscripten"
 pure-eval==0.2.2
 pyasn1==0.6.0
 pyasn1-modules==0.4.0
-pycparser==2.22; implementation_name != "cpython"
+pycparser==2.22; platform_python_implementation != "PyPy" or implementation_name != "cpython"
 pycryptodomex==3.20.0
 pydantic==2.7.1
 pydantic-core==2.18.2
-pygments==2.17.2
+pygments==2.18.0
 python-crontab==3.0.0
 python-dateutil==2.9.0.post0
 python-ldap==3.4.4
 pytz==2024.1
-regex==2024.4.16
+regex==2024.4.28
 requests==2.31.0
 setuptools==69.5.1
 sgmllib3k==1.0.0
 six==1.16.0
+sniffio==1.3.1
 sonic-client==1.0.0
 sqlparse==0.5.0
 stack-data==0.6.3

Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ dependencies = [`
`35`	`35`	`# - gallery-dl`
`36`	`36`	`# - scihubdl`
`37`	`37`	`# - See Github issues for more...`
	`38`	`+ "django-signal-webhooks>=0.3.0",`
`38`	`39`	`]`
`39`	`40`
`40`	`41`	`homepage = "https://github.com/ArchiveBox/ArchiveBox"`