AWS FC2023 envfile

ingest/fc/prod/deploy-fc-kafka.sh

@@ -1,19 +1,41 @@
 #!/bin/sh
-
 set -e
+DEBUG=1
+
 
+# read environment file
 if [[ "$1" != "" ]]; then
-    ENV_TAG="$1"
+	ENVFILE="$1"
 else
-    echo "You must give an argument that specifies the deployment, e.g. crawler06 uses prod-env-crawler06.sh."
-    exit 1
+	echo "ERROR You must give an argument that specifies the deployment, e.g. crawler06 uses prod-env-crawler06.sh."
+	exit 1
 fi
+if ! [[ -f ${ENVFILE} ]]; then
+	echo "ERROR: argument [${ENVFILE}] environment file missing"
+	exit 1
+fi
+source ./${ENVFILE}
 
 
-source ./env-${ENV_TAG}.sh
+# check STORAGE_PATH exists, create any missing sub-directories
+if !  [[ -d ${STORAGE_PATH} ]]; then
+	echo "ERROR: STORAGE_PATH [${STORAGE_PATH}] defined in [${ENVFILE}] missing"
+	exit 1
+fi
+for _d in ${TMP_STORAGE_PATH} ${CDX_STORAGE_PATH} ${ZK_DATA_PATH} ${ZK_DATALOG_PATH} ${KAFKA_PATH}; do
+	if [[ "${_d}" == "" ]]; then
+		echo "ERROR: No directory defined"
+		exit 1
+	fi
+	if ! [[ -d ${_d} ]]; then
+		[[ ${DEBUG} ]] && echo -e "DEBUG]\t making dir [${_d}]"
+		mkdir -p ${_d} || {
+			echo "ERROR: failed to make directory [${_d}]"
+			exit 1
+		}
+	fi
+done
 
-mkdir -p ${STORAGE_PATH}/zookeeper/data
-mkdir -p ${STORAGE_PATH}/zookeeper/datalog
-mkdir -p ${STORAGE_PATH}/kafka
 
+# start FC kafka stack
 docker stack deploy -c ../fc-kafka/docker-compose.yml fc_kafka

ingest/fc/prod/env-aws-fc2023-prod.sh

@@ -0,0 +1,11 @@
+# Common directories
+export STORAGE_PATH=/mnt/data/fc
+export TMP_STORAGE_PATH=${STORAGE_PATH}/tmp
+export CDX_STORAGE_PATH=${STORAGE_PATH}/cdx
+export ZK_DATA_PATH=${STORAGE_PATH}/zookeeper/data
+export ZK_DATALOG_PATH=${STORAGE_PATH}/zookeeper/datalog
+export KAFKA_PATH=${STORAGE_PATH}/kafka
+
+# crawler details
+export CRAWL_HOST_LAN_IP=172.31.43.254
+export CRAWL_HOST_WAN_IP=18.130.205.6