This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Model Upload with Versioning and Cleanup | |
| on: | |
| push: | |
| paths: | |
| - "s3-uploads/**" # Trigger on changes in the 's3-uploads' directory | |
| - "model_versions.json" # Trigger on changes in the version file | |
| jobs: | |
| upload_model: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up AWS CLI | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| - name: Compare model versions | |
| id: compare_versions | |
| run: | | |
| # Read the current and previous model versions | |
| CURRENT_VERSIONS=$(cat model_versions.json) | |
| S3_BUCKET="${{ secrets.S3_BUCKET_NAME }}" | |
| CHANGED_MODELS="" | |
| for MODEL in $(jq -r 'keys[]' <<< "$CURRENT_VERSIONS"); do | |
| CURRENT_VERSION=$(jq -r --arg model "$MODEL" '.[$model]' <<< "$CURRENT_VERSIONS") | |
| S3_FILE="s3://$S3_BUCKET/${MODEL}-${CURRENT_VERSION}" | |
| # Check if the model with the current version already exists in S3 | |
| if ! aws s3 ls "$S3_FILE" >/dev/null 2>&1; then | |
| echo "New or updated model detected: $MODEL (version $CURRENT_VERSION)" | |
| CHANGED_MODELS="$CHANGED_MODELS $MODEL" | |
| fi | |
| done | |
| echo "changed_models=$CHANGED_MODELS" >> $GITHUB_ENV | |
| - name: Upload private key to S3 (from s3-uploads folder) | |
| continue-on-error: true # Prevent workflow failure if key upload fails | |
| run: | | |
| PRIVATE_KEY_PATH="s3-uploads/private_key.pem" | |
| S3_BUCKET="${{ secrets.S3_BUCKET_NAME }}" | |
| KMS_KEY_ID="${{ secrets.KMS_KEY_ID }}" | |
| if [ -f "$PRIVATE_KEY_PATH" ]; then | |
| echo "Found private key at $PRIVATE_KEY_PATH" | |
| # Upload private key to S3 with encryption | |
| echo "Uploading private key to S3..." | |
| if aws s3 cp "$PRIVATE_KEY_PATH" "s3://$S3_BUCKET/keys/private_key.pem" --sse aws:kms --sse-kms-key-id "$KMS_KEY_ID"; then | |
| echo "Successfully uploaded private key to S3" | |
| # After successful upload, delete the private key from the repository | |
| echo "Deleting private key from the repository after uploading to S3..." | |
| rm "$PRIVATE_KEY_PATH" | |
| else | |
| echo "Failed to upload private key to S3" | |
| fi | |
| else | |
| echo "Private key not found at $PRIVATE_KEY_PATH - Skipping upload" | |
| fi | |
| - name: Upload updated models | |
| if: env.changed_models != '' | |
| run: | | |
| CHANGED_MODELS="${{ env.changed_models }}" | |
| CURRENT_VERSIONS=$(cat model_versions.json) | |
| S3_BUCKET="${{ secrets.S3_BUCKET_NAME }}" | |
| KMS_KEY_ID="${{ secrets.KMS_KEY_ID }}" | |
| for MODEL in $CHANGED_MODELS; do | |
| if [ -z "$MODEL" ]; then | |
| echo "Skipping empty model entry." | |
| continue | |
| fi | |
| MODEL_PATH="s3-uploads/$MODEL" | |
| CURRENT_VERSION=$(jq -r --arg model "$MODEL" '.[$model]' <<< "$CURRENT_VERSIONS") | |
| S3_FILE="s3://$S3_BUCKET/${MODEL}-${CURRENT_VERSION}" | |
| if [ -f "$MODEL_PATH" ]; then | |
| echo "Uploading $MODEL (version $CURRENT_VERSION) to $S3_FILE" | |
| # Upload to S3 with SSE-KMS | |
| aws s3 cp "$MODEL_PATH" "$S3_FILE" --sse aws:kms --sse-kms-key-id "$KMS_KEY_ID" | |
| # Check if upload was successful, then delete the file from the repo | |
| if [ $? -eq 0 ]; then | |
| echo "Model uploaded successfully, deleting $MODEL from the repository." | |
| rm "$MODEL_PATH" | |
| else | |
| echo "Model upload failed for $MODEL." | |
| exit 1 | |
| fi | |
| else | |
| echo "Model file $MODEL_PATH not found. Skipping upload." | |
| fi | |
| done | |
| - name: Empty the s3-uploads folder | |
| if: env.changed_models != '' | |
| run: | | |
| echo "Cleaning up the s3-uploads folder..." | |
| rm -rf s3-uploads/* | |
| - name: Commit model deletions and changes | |
| if: env.changed_models != '' | |
| run: | | |
| git config --global user.name "github-actions" | |
| git config --global user.email "[email protected]" | |
| git remote set-url origin https://x-access-token:${{ secrets.PAT_TOKEN }}@github.com/${{ github.repository }}.git | |
| git add -u | |
| git commit -m "Delete uploaded models and private key from repository after S3 upload" | |
| git push |