ArchiveBox
diff --git a/‎.dockerignore
Copy file name to clipboard
+9-13Lines changed: 9 additions & 13 deletions b/‎.dockerignore
Copy file name to clipboard
+9-13Lines changed: 9 additions & 13 deletions
diff --git a/‎.flake8
Copy file name to clipboard
+6Lines changed: 6 additions & 0 deletions b/‎.flake8
Copy file name to clipboard
+6Lines changed: 6 additions & 0 deletions
diff --git a/‎.github/workflows/test.yml
Copy file name to clipboard
+145Lines changed: 145 additions & 0 deletions b/‎.github/workflows/test.yml
Copy file name to clipboard
+145Lines changed: 145 additions & 0 deletions
diff --git a/‎.gitignore
Copy file name to clipboard
+9-14Lines changed: 9 additions & 14 deletions b/‎.gitignore
Copy file name to clipboard
+9-14Lines changed: 9 additions & 14 deletions
diff --git a/‎Dockerfile
Copy file name to clipboard
+56-62Lines changed: 56 additions & 62 deletions b/‎Dockerfile
Copy file name to clipboard
+56-62Lines changed: 56 additions & 62 deletions
diff --git a/‎MANIFEST.in
Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions b/‎MANIFEST.in
Copy file name to clipboard
+4Lines changed: 4 additions & 0 deletions
diff --git a/‎Pipfile
Copy file name to clipboard
+12Lines changed: 12 additions & 0 deletions b/‎Pipfile
Copy file name to clipboard
+12Lines changed: 12 additions & 0 deletions
@@ -1,20 +1,16 @@
 .DS_Store
 ._*
-
+*.pyc
 __pycache__/
 .mypy_cache/
-archivebox.egg-info/
-build/
-dist/
 
-# Dependency code
-.venv                # main pipenv venv path
-venv                 # old venv path, (no longer used)
-archivebox/.venv     # old venv path, (no longer used)
-archivebox/venv      # old venv path, (no longer used)
+venv/
+.venv/
+.docker-venv/
 
+*.egg-info/
+build/
+dist/
 
-# Stateful data folders
-data                 # main archivebox data folder
-archivebox/output    # old output folder path (no longer used)
-output               # old output folder path (no longer used)
+data/
+output/
@@ -0,0 +1,6 @@
+[flake8]
+ignore = D100,D101,D102,D103,D104,D105,D202,D203,D205,D400,E131,E241,E252,E266,E272,E701,E731,W293,W503,W291,W391
+select = F,E9,W
+max-line-length = 130
+max-complexity = 10
+exclude = migrations,tests,node_modules,vendor,venv,.venv,.venv2,.docker-venv
@@ -0,0 +1,145 @@
+name: Test workflow
+on: [push]
+
+env:
+  MAX_LINE_LENGTH: 110
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+          architecture: x64
+
+      - name: Install flake8
+        run: |
+          pip install flake8
+
+      - name: Lint with flake8
+        run: |
+          # one pass for show-stopper syntax errors or undefined names
+          flake8 archivebox --count --show-source --statistics
+          # one pass for small stylistic things
+          flake8 archivebox --count --max-line-length="$MAX_LINE_LENGTH" --statistics
+
+      # - name: Lint with mypy
+      #   run: |
+      #     pip install mypy
+      #     mypy archivebox || true
+
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: [3.7, 3.8]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Cache virtualenv
+        uses: actions/cache@v2
+        id: cache-venv
+        with:
+          path: .venv
+          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python }}-venv-
+
+      - name: Create virtualenv
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          python3 -m venv .venv
+          source .venv/bin/activate
+          python3 -m pip install --upgrade pip setuptools
+
+      - name: Install dependencies
+        run: |
+          source .venv/bin/activate
+          python -m pip install .
+          python -m pip install pytest bottle
+
+      - name: Test built package with pytest
+        run: |
+          source .venv/bin/activate
+          python -m pytest -s
+
+  docker-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - uses: satackey/action-docker-layer-caching@v0.0.4
+
+      - name: Build image
+        run: |
+          docker build . -t archivebox
+
+      - name: Init data dir
+        run: |
+          mkdir data
+          docker run -v "$PWD"/data:/data archivebox init
+
+      - name: Run test server
+        run: |
+          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
+          docker run --name www-nginx -p 80:80 -d nginx
+
+      - name: Add link
+        run: |
+          docker run -v "$PWD"/data:/data --network host archivebox add https://www.test-nginx-1.local
+
+      - name: Add stdin link
+        run: |
+          echo "https://www.test-nginx-2.local" | docker run -i -v "$PWD"/data:/data archivebox add
+
+      - name: List links
+        run: |
+          docker run -v "$PWD"/data:/data archivebox list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
+          docker run -v "$PWD"/data:/data archivebox list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }
+
+
+      - name: Start docker-compose stack
+        run: |
+          docker-compose run archivebox init
+          docker-compose up -d
+          sleep 4
+          curl --silent --location 'https://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'django.jQuery'
+
+      - name: Check added urls show up in index
+        run: |
+          docker-compose run archivebox add 'https://example.com/#test_docker' --index-only
+          curl --silent --location 'https://127.0.0.1:8000' | grep 'https://example.com/#test_docker'
+
+      - name: Curl index with PUBLIC_INDEX=False
+        run: |
+          docker-compose run archivebox config --set PUBLIC_INDEX=False
+          docker-compose up -d || true
+          sleep 8
+          curl --silent --location 'https://127.0.0.1:8000' | grep 'Log in'
+          docker-compose down
+
+      - name: Curl index with PUBLIC_INDEX=True
+        run: |
+          docker-compose run archivebox config --set PUBLIC_INDEX=True
+          docker-compose up -d || true
+          sleep 8
+          curl --silent --location 'https://127.0.0.1:8000' | grep 'Add Links'
+          docker-compose down
@@ -1,21 +1,16 @@
 .DS_Store
 ._*
-
+*.pyc
 __pycache__/
 .mypy_cache/
-archivebox.egg-info/
-build/
-dist/
 
+venv/
+.venv/
+.docker-venv/
 
-# Dependency code
-.venv                # main pipenv venv path
-venv                 # old venv path, (no longer used)
-archivebox/.venv     # old venv path, (no longer used)
-archivebox/venv      # old venv path, (no longer used)
-
+*.egg-info/
+build/
+dist/
 
-# Stateful data folders
-data/                # main archivebox data folder
-archivebox/output/   # old output folder path (no longer used)
-output/              # old output folder path (no longer used)
+data/
+output/
@@ -1,74 +1,68 @@
-# This Dockerfile for ArchiveBox installs the following in a container:
-#     - curl, wget, python3, youtube-dl, google-chrome-unstable
-#     - ArchiveBox
+# This is the Dockerfile for ArchiveBox, it includes the following major pieces:
+#     git, curl, wget, python3, youtube-dl, google-chrome-stable, ArchiveBox
 # Usage:
-#     docker build github.com/pirate/ArchiveBox -t archivebox
-#     echo 'https://example.com' | docker run -i -v ./data:/data archivebox /bin/archive
-#     docker run -v ./data:/data archivebox /bin/archive 'https://example.com/some/rss/feed.xml'
+#     docker build . -t archivebox
+#     docker run -v "$PWD/data":/data archivebox init
+#     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
 # Documentation:
 #     https://github.com/pirate/ArchiveBox/wiki/Docker#docker
 
-# TODO: bump to latest chrome and node version, confirm chrome doesn't hang on simple pages
+FROM python:3.8-slim-buster
 
-FROM node:11-slim
-LABEL maintainer="Nick Sweeting <archivebox-git@sweeting.me>"
+LABEL name="archivebox" \
+      maintainer="Nick Sweeting <archivebox-git@sweeting.me>" \
+      description="All-in-one personal internet archiving container"
 
-RUN apt-get update \
-    && apt-get install -yq --no-install-recommends \
-        git wget curl youtube-dl gnupg2 libgconf-2-4 python3 python3-pip \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install latest chrome package and fonts to support major charsets (Chinese, Japanese, Arabic, Hebrew, Thai and a few others)
-RUN apt-get update && apt-get install -y wget --no-install-recommends \
-    && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
-    && sh -c 'echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
-    && apt-get update \
-    && apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst ttf-freefont \
-      --no-install-recommends \
-    && rm -rf /var/lib/apt/lists/* \
-    && rm -rf /src/*.deb
-
-# It's a good idea to use dumb-init to help prevent zombie chrome processes.
-ADD https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 /usr/local/bin/dumb-init
-RUN chmod +x /usr/local/bin/dumb-init
-
-# Uncomment to skip the chromium download when installing puppeteer. If you do,
-# you'll need to launch puppeteer with:
-#     browser.launch({executablePath: 'google-chrome-unstable'})
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
-
-# Install puppeteer so it's available in the container.
-RUN npm i puppeteer
-
-# Add user so we don't need --no-sandbox.
-RUN groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \
-    && mkdir -p /home/pptruser/Downloads \
-    && chown -R pptruser:pptruser /home/pptruser \
-    && chown -R pptruser:pptruser /node_modules
-
-# Install the ArchiveBox repository and pip requirements
-COPY . /home/pptruser/app
-RUN mkdir -p /data \
-    && chown -R pptruser:pptruser /data \
-    && ln -s /data /home/pptruser/app/archivebox/output \
-    && ln -s /home/pptruser/app/bin/* /bin/ \
-    && ln -s /home/pptruser/app/bin/archivebox /bin/archive \
-    && chown -R pptruser:pptruser /home/pptruser/app/archivebox
-    # && pip3 install -r /home/pptruser/app/archivebox/requirements.txt
-
-VOLUME /data
-
-ENV LANG=C.UTF-8 \
+ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 \
     PYTHONIOENCODING=UTF-8 \
-    CHROME_SANDBOX=False \
-    CHROME_BINARY=google-chrome-unstable \
-    OUTPUT_DIR=/data
+    PYTHONUNBUFFERED=1 \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    CODE_PATH=/app \
+    VENV_PATH=/venv \
+    DATA_PATH=/data
+
+# First install CLI utils and base deps, then Chrome + Fons
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+       apt-transport-https ca-certificates apt-utils gnupg gosu gnupg2 libgconf-2-4 zlib1g-dev \
+       dumb-init jq git wget curl youtube-dl ffmpeg \
+    && curl -sSL "https://dl.google.com/linux/linux_signing_key.pub" | apt-key add - \
+    && echo "deb https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
+    && apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+       google-chrome-stable \
+       fontconfig \
+       fonts-ipafont-gothic \
+       fonts-wqy-zenhei \
+       fonts-thai-tlwg \
+       fonts-kacst \
+       fonts-symbola \
+       fonts-noto \
+       fonts-freefont-ttf \
+    && rm -rf /var/lib/apt/lists/*
 
 # Run everything from here on out as non-privileged user
-USER pptruser
-WORKDIR /home/pptruser/app
+RUN groupadd --system archivebox \
+    && useradd --system --create-home --gid archivebox --groups audio,video archivebox
+
+ADD . "$CODE_PATH"
+WORKDIR "$CODE_PATH"
+ENV PATH="${PATH}:$VENV_PATH/bin"
+RUN python -m venv --clear --symlinks "$VENV_PATH" \
+    && pip install --upgrade pip setuptools \
+    && pip install -e .
+
+VOLUME "$DATA_PATH"
+WORKDIR "$DATA_PATH"
+EXPOSE 8000
+ENV CHROME_BINARY=google-chrome \
+    CHROME_SANDBOX=False
+
+RUN env ALLOW_ROOT=True archivebox version
 
-ENTRYPOINT ["dumb-init", "--"]
-CMD ["/bin/archive"]
+ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh", "archivebox"]
+CMD ["server", "0.0.0.0:8000"]
@@ -0,0 +1,4 @@
+include LICENSE
+include README.md
+include archivebox/VERSION
+recursive-include archivebox/themes *
@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[packages]
+# see setup.py for package dependency list
+"e1839a8" = {path = ".", editable = true}
+
+[dev-packages]
+# see setup.py for dev package dependency list
+"e1839a8" = {path = ".", extras = ["dev"], editable = true}