feat: multi-arch

README.md

@@ -117,11 +117,11 @@ echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.
 echo 'eval "$(pyenv init -)"' >> ~/.bashrc
 source ~/.bashrc
 
-# Install Python 3.10 inside pyenv:
-pyenv install 3.10
+# Install Python 3.11 inside pyenv:
+pyenv install 3.11
 
 # Set the active version of Python:
-pyenv local 3.10
+pyenv local 3.11
 
 # Upgrade pip:
 pip install --upgrade pip
@@ -139,27 +139,16 @@ source ~/.bashrc
 
 # Ensure that you are in the "typesense-docsearch-scraper" directory.
 # Then, install the Python dependencies for this project:
-pipenv --python 3.10
+pipenv --python 3.11
 pipenv lock --clear
 pipenv install
 
 # Then, open a shell with with the Python environment:
 pipenv shell
 
-# Build a new version of the base Docker container - ONLY NEEDED WHEN WE CHANGE DEPENDENCIES
-export SCRAPER_BASE_VERSION="0.7.0" # Only need to change this when we update dependencies
-docker buildx build -f ./scraper/dev/docker/Dockerfile.base -t typesense/docsearch-scraper-base:${SCRAPER_BASE_VERSION} .
-docker push typesense/docsearch-scraper-base:${SCRAPER_BASE_VERSION}
-docker tag typesense/docsearch-scraper-base:${SCRAPER_BASE_VERSION} typesense/docsearch-scraper-base:latest
-docker push typesense/docsearch-scraper-base:latest
-
 # Build a new version of the scraper Docker container
 export SCRAPER_VERSION="0.9.1"
-export SCRAPER_BASE_VERSION="latest"
-docker buildx build -f ./scraper/dev/docker/Dockerfile --build-arg SCRAPER_BASE_VERSION=${SCRAPER_BASE_VERSION} -t typesense/docsearch-scraper:${SCRAPER_VERSION} .
-docker push typesense/docsearch-scraper:${SCRAPER_VERSION}
-docker tag typesense/docsearch-scraper:${SCRAPER_VERSION} typesense/docsearch-scraper:latest
-docker push typesense/docsearch-scraper:latest
+docker buildx build -f ./scraper/dev/docker/Dockerfile -t typesense/docsearch-scraper:${SCRAPER_VERSION} -t typesense/docsearch-scraper:latest . --target release --platform linux/amd64,linux/arm64 --push
 
 # Add a new Git tag.
 git tag -a "${SCRAPER_VERSION}" -m "${SCRAPER_VERSION}"

cli/src/commands/abstract_build_docker.py

@@ -3,16 +3,20 @@
 
 class AbstractBuildDocker(AbstractCommand):
     @staticmethod
-    def build_docker_file(file, image="typesense/docsearch-scraper-dev",
-                          local_tag=False):
+    def build_docker_target(file, image="typesense/docsearch-scraper-dev",
+                            local_tag=False):
         tags = [image]
 
         if local_tag:
             tag = AbstractBuildDocker.get_local_tag().decode()
             tags.append(image + ":" + tag)
 
-        cmd = ["docker", "build"] + [param for tag in tags for param in
-                                     ['-t', tag]] + ["-f", file, "."]
+        cmd = ["docker", "buildx", "build"] + [param for tag in tags for param in
+                                     ['-t', tag]] + ["-f", "scraper/dev/docker/Dockerfile", "--target", file, "."]
+        if local_tag:
+            cmd += ["--platform", "linux/amd64,linux/arm64", "--push"]
+        else:
+            cmd += ["--load"]
         return AbstractCommand.exec_shell_command(cmd)
 
     def get_options(self):

cli/src/commands/build_docker_scraper.py

@@ -23,19 +23,19 @@ def run(self, args):
         test = self.get_option("test", args)
 
         if test:
-            return self.build_docker_file("scraper/dev/docker/Dockerfile.test",
+            return self.build_docker_target("test",
                                           "typesense/docsearch-scraper-test",
-                                          local_tag=local_tag)
+                                            local_tag=local_tag)
 
-        code = self.build_docker_file("scraper/dev/docker/Dockerfile.base",
+        code = self.build_docker_target("base",
                                       "typesense/docsearch-scraper-base",
-                                      local_tag=local_tag)
+                                        local_tag=local_tag)
 
         if code != 0:
             return code
-        code = self.build_docker_file("scraper/dev/docker/Dockerfile.dev",
-                                      local_tag=local_tag)
+        code = self.build_docker_target("dev",
+                                        local_tag=local_tag)
         if code != 0:
             return code
-        return self.build_docker_file("scraper/dev/docker/Dockerfile",
+        return self.build_docker_target("release",
                                       "typesense/docsearch-scraper", local_tag=local_tag)

scraper/dev/docker/Dockerfile

@@ -1,5 +1,72 @@
-ARG SCRAPER_BASE_VERSION
-FROM typesense/docsearch-scraper-base:$SCRAPER_BASE_VERSION
+# syntax=docker/dockerfile:1.4
+FROM debian:12-slim as base
+LABEL maintainer="[email protected]"
+
+# Install selenium
+ENV LC_ALL C
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+RUN useradd -d /home/seleuser -m seleuser
+RUN chown -R seleuser /home/seleuser
+RUN chgrp -R seleuser /home/seleuser
+
+WORKDIR /home/seleuser
+
+RUN apt-get update -y && apt-get install -yq \
+    software-properties-common\
+    python3
+
+RUN apt-get update -y && apt-get install -yq \
+    curl \
+    wget \
+    sudo \
+    gnupg \
+    && curl -sL https://deb.nodesource.com/setup_18.x | sudo bash -
+RUN apt-get update -y && apt-get install -y \
+    nodejs
+RUN apt-get update -y && apt-get install -yq \
+  unzip \
+  xvfb \
+  libxi6 \
+  libgconf-2-4 \
+  default-jdk
+
+RUN apt-get update -y && apt-get install -yq \
+  chromium-driver
+
+RUN wget -q https://github.com/SeleniumHQ/selenium/releases/download/selenium-4.4.0/selenium-server-4.4.0.jar
+RUN wget -q https://repo1.maven.org/maven2/org/testng/testng/7.6.1/testng-7.6.1.jar
+
+# Install DocSearch dependencies
+COPY Pipfile .
+COPY Pipfile.lock .
+
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
+ENV PIPENV_HIDE_EMOJIS 1
+RUN apt-get update -y && apt-get install -yq \
+    python3-pip
+RUN pip3 install pipenv --break-system-packages
+
+USER 1000
+RUN pipenv sync --python 3.11
+
+FROM base AS dev
+LABEL maintainer="[email protected]"
+
+FROM base as test
+LABEL maintainer="[email protected]"
+
+WORKDIR /home/seleuser
+USER 1000
+
+# Copy DocSearch files
+COPY . .
+RUN touch .env
+ENTRYPOINT ["pipenv", "run", "./docsearch", "test", "no_browser"]
+
+FROM base as release
 LABEL maintainer="[email protected]"
 
 WORKDIR /home/seleuser