Merge remote-tracking branch 'refs/remotes/upstream/main'

# Conflicts:
#	chromadb/utils/embedding_functions.py

.github/DEVELOP.md

@@ -0,0 +1,6 @@
+GitHub *still* does not support organizing workflows into directories, so instead we use some notation:
+
+- A workflow starting with `_` is a reusable workflow and should exclusively have a `workflow_call` trigger.
+- Any other workflow is expected to have standard triggers (e.g. `push`, `pull_request`, etc.) and should not be called by other workflows.
+
+All workflows should be prefixed by their language name, e.g. `python-test.yml`.

.github/actions/docker/action.yaml

@@ -0,0 +1,37 @@
+name: Setup Docker
+description: "This action sets up Docker Buildx and authenticates to registries"
+inputs:
+  ghcr-username:
+    description: "Github Container Registry username"
+    required: true
+  ghcr-password:
+    description: "Github Container Registry password"
+    required: true
+  dockerhub-username:
+    description: "DockerHub username"
+    required: true
+  dockerhub-password:
+    description: "DockerHub password"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    # https://github.com/docker/setup-qemu-action - for multiplatform builds
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+    # https://github.com/docker/setup-buildx-action - for multiplatform builds
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v2
+    - name: Log in to the Github Container registry
+      uses: docker/[email protected]
+      with:
+        registry: ghcr.io
+        username: ${{ inputs.ghcr-username }}
+        password: ${{ inputs.ghcr-password }}
+    - name: Login to DockerHub
+      uses: docker/[email protected]
+      with:
+        username: ${{ inputs.dockerhub-username }}
+        password: ${{ inputs.dockerhub-password }}

.github/actions/go/action.yaml

@@ -0,0 +1,9 @@
+name: Setup Go
+description: "This action sets up Go"
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-go@v5
+      with:
+        cache-dependency-path: go/go.sum
+    - uses: ariga/setup-atlas@v0

.github/actions/python/action.yaml

@@ -0,0 +1,23 @@
+name: Setup Python
+description: "This action sets up Python and installs dependencies"
+inputs:
+  python-version:
+    description: "Python version to use"
+    required: false
+    default: "3.8"
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: "pip"
+        cache-dependency-path: "requirements*.txt"
+    - name: Install test dependencies
+      run: python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
+      shell: bash
+    - name: Upgrade SQLite
+      run: python bin/windows_upgrade_sqlite.py
+      shell: bash
+      if: runner.os == 'Windows'

.github/actions/rust/action.yaml

@@ -0,0 +1,12 @@
+name: Setup Rust
+description: "This action sets up Rust"
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout chroma-hnswlib
+      uses: actions/checkout@v3
+      with:
+        repository: chroma-core/hnswlib
+        path: hnswlib
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v2

.github/actions/tilt/action.yaml

@@ -0,0 +1,23 @@
+name: Start Tilt services
+description: "This action starts Tilt services"
+runs:
+  using: "composite"
+  steps:
+    - name: Install Tilt
+      shell: bash
+      run: |
+        TILT_VERSION="0.33.3"
+        curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin tilt
+    - name: Install ctlptlc
+      shell: bash
+      run: |
+        CTLPTL_VERSION="0.8.20"
+        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
+          tar -xzv -C /usr/local/bin ctlptl
+    - name: Set up kind
+      shell: bash
+      run: ctlptl create cluster kind --registry=ctlptl-registry
+    - name: Start Tilt
+      shell: bash
+      run: tilt ci

.github/workflows/_go-tests.yml

@@ -0,0 +1,46 @@
+name: Go tests
+
+on:
+  workflow_call:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: chroma
+          POSTGRES_PASSWORD: chroma
+          POSTGRES_DB: chroma
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup
+        uses: ./.github/actions/go
+      - name: Build and test
+        run: make test
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+        working-directory: go
+
+  cluster-test:
+    runs-on: "16core-64gb-ubuntu-latest"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup
+        uses: ./.github/actions/go
+      - uses: ./.github/actions/tilt
+      - run: bin/cluster-test.sh bash -c 'cd go && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma-core/chroma/go/pkg/memberlist_manager'

.github/workflows/_javascript-client-tests.yml

@@ -0,0 +1,13 @@
+name: JavaScript client tests
+
+on:
+  workflow_call:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Test
+        run: bin/ts-integration-test

.github/workflows/_python-tests.yml

@@ -0,0 +1,167 @@
+name: Chroma Python Base Tests
+
+on:
+  workflow_call:
+    inputs:
+      python_versions:
+        description: 'Python versions to test (as json array)'
+        required: false
+        default: '["3.8"]'
+        type: string
+      property_testing_preset:
+        description: 'Property testing preset'
+        required: true
+        type: string
+
+jobs:
+  test:
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ${{fromJson(inputs.python_versions)}}
+        platform: [ubuntu-latest, windows-latest]
+        test-globs: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore-glob 'chromadb/test/distributed/*' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
+                   "chromadb/test/auth/test_simple_rbac_authz.py",
+                   "chromadb/test/property/test_add.py",
+                   "chromadb/test/property/test_collections.py",
+                   "chromadb/test/property/test_collections_with_database_tenant.py",
+                   "chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
+                   "chromadb/test/property/test_cross_version_persist.py",
+                   "chromadb/test/property/test_embeddings.py",
+                   "chromadb/test/property/test_filtering.py",
+                   "chromadb/test/property/test_persist.py"]
+        include:
+          - test-globs: "chromadb/test/property/test_embeddings.py"
+            parallelized: true
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup
+        uses: ./.github/actions/python
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Test
+        run: python -m pytest ${{ matrix.parallelized && '-n auto' || '' }} ${{ matrix.test-globs }}
+        shell: bash
+        env:
+          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
+
+  # todo: break out stress integration tests
+  test-single-node-integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ${{fromJson(inputs.python_versions)}}
+        platform: [ubuntu-latest, windows-latest]
+        test-globs: ["--ignore-glob 'chromadb/test/property/*' --ignore='chromadb/test/test_cli.py' --ignore-glob 'chromadb/test/distributed/*' --ignore='chromadb/test/auth/test_simple_rbac_authz.py'",
+                   "chromadb/test/property/test_add.py",
+                   "chromadb/test/test_cli.py",
+                   "chromadb/test/auth/test_simple_rbac_authz.py",
+                   "chromadb/test/property/test_collections.py",
+                  "chromadb/test/property/test_collections_with_database_tenant.py",
+                   "chromadb/test/property/test_cross_version_persist.py",
+                   "chromadb/test/property/test_embeddings.py",
+                   "chromadb/test/property/test_filtering.py",
+                   "chromadb/test/property/test_persist.py"]
+        include:
+          - platform: ubuntu-latest
+            env-file: compose-env.linux
+          - platform: windows-latest
+            env-file: compose-env.windows
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python (${{ matrix.python }})
+      uses: ./.github/actions/python
+    - name: Integration Test
+      run: bin/python-integration-test ${{ matrix.test-globs }}
+      shell: bash
+      env:
+        ENV_FILE: ${{ matrix.env-file }}
+        PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
+
+  test-cluster-integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ${{fromJson(inputs.python_versions)}}
+        platform: ["16core-64gb-ubuntu-latest"]
+        test-globs: ["chromadb/test/db/test_system.py",
+                   "chromadb/test/property/test_collections.py",
+                   "chromadb/test/property/test_add.py",
+                   "chromadb/test/property/test_filtering.py",
+                   "chromadb/test/property/test_embeddings.py",
+                   "chromadb/test/property/test_collections_with_database_tenant.py",
+                   "chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
+                   "chromadb/test/ingest/test_producer_consumer.py",
+                   "chromadb/test/segment/distributed/test_memberlist_provider.py",
+                   "chromadb/test/test_logservice.py",
+                   "chromadb/test/distributed/test_sanity.py"]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/python
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: ./.github/actions/tilt
+      - name: Test
+        run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}"'
+        shell: bash
+        env:
+          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
+
+  test-thin-client:
+    strategy:
+      matrix:
+        python: ${{fromJson(inputs.python_versions)}}
+        platform: [ubuntu-latest] # # todo: should run on Windows, currently failing because Dockerfile doesn't build
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup
+        uses: ./.github/actions/python
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Test
+        run: clients/python/integration-test.sh
+        shell: bash
+        env:
+          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
+
+  test-stress:
+    timeout-minutes: 90
+    strategy:
+      matrix:
+        python: ${{fromJson(inputs.python_versions)}}
+        platform: ['16core-64gb-ubuntu-latest', '16core-64gb-windows-latest']
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/python
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Test
+        run: python -m pytest chromadb/test/stress/
+        shell: bash
+        env:
+          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
+
+  test-single-node-integration-stress:
+    strategy:
+      matrix:
+        python: ${{fromJson(inputs.python_versions)}}
+        platform: [ubuntu-latest] # todo: should run on Windows, currently failing because Dockerfile doesn't build
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Set up Python (${{ matrix.python }})
+      uses: ./.github/actions/python
+    - name: Integration Test
+      run: bin/python-integration-test chromadb/test/stress/
+      shell: bash
+      env:
+        PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

.github/workflows/python-vuln.yaml → .github/workflows/_python-vulnerability-scan.yml

@@ -1,19 +1,15 @@
-name: Python Vulnerability Scan
+name: Scan for Python Vulnerabilities
+
 on:
-  push:
-    branches:
-      - '*'
-      - '*/**'
-    paths:
-      - chromadb/**
-      - clients/python/**
-  workflow_dispatch:
+  workflow_call:
+
 jobs:
   bandit-scan:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v3
+      - name: Setup
+        uses: ./.github/actions/python
       - uses: ./.github/actions/bandit-scan/
         with:
           input-dir: '.'

.github/workflows/_rust-tests.yml

@@ -0,0 +1,25 @@
+name: Rust tests
+
+on:
+  workflow_call:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+    runs-on: ${{ matrix.platform }}
+    defaults:
+      run:
+        working-directory: chroma
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: chroma
+      - name: Setup
+        uses: ./chroma/.github/actions/rust
+      - name: Build
+        run: cargo build --verbose
+      - name: Test
+        run: cargo test --verbose