-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathdeployment_profiles.cfg
82 lines (69 loc) · 3.25 KB
/
deployment_profiles.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
[DEFAULT]
deployment_environment: local
deps_dir: %(PWD)s/_dependencies
#jars: %(deps_dir)s/libs/aws-java-sdk-1.7.4.jar,%(deps_dir)s/libs/hadoop-aws-2.7.0.jar,%(deps_dir)s/libs/spark-cassandra-connector_2.10-1.6.0-M1.jar,%(deps_dir)s/libs/spark-streaming-kafka_2.10-1.6.0.jar,%(deps_dir)s/cassandra-java-driver-2.1.1/cassandra-driver-core-2.1.1.jar,%(deps_dir)s/libs/pyspark-cassandra-0.2.1.jar
jars: %(deps_dir)s/libs/aws-java-sdk-1.7.4.jar,%(deps_dir)s/libs/hadoop-aws-2.7.0.jar,%(deps_dir)s/libs/spark-streaming-kafka_2.10-1.6.0.jar
# Includes all the jars here and more
jars_dirs: %(deps_dir)s/kafka_2.10-0.8.2.2/libs
#jars_dirs: %(deps_dir)s/kafka_2.10-0.8.2.2/libs,%(deps_dir)s/cassandra-java-driver-2.1.1/lib
#py_files: %(deps_dir)s/libs/pyspark-cassandra-0.2.1.jar,%(deps_dir)s/pyspark-cassandra-0.2.1/python
#py_files: %(deps_dir)s/libs/pyspark-cassandra-0.2.1.jar
master: local[2]
# Override this if you want to use a different path
# If you brew installed it, it'll be in /usr/local/Cellar/mesos/0.25.0
mesos_libs_dir: %(deps_dir)s/mesos-0.25.0/build/src/.libs
tmp_dir: _tmp
history_port: 18080
enable_event_logs: true
metrics_conf: conf/metrics.properties
py_files:
packages:
projects_dir: projects
docs_dir: doc
[local]
master: local[4]
jars: %(deps_dir)s/libs/aws-java-sdk-1.7.4.jar,%(deps_dir)s/libs/hadoop-aws-2.7.0.jar,%(deps_dir)s/libs/spark-cassandra-connector_2.10-1.6.0-M1.jar,%(deps_dir)s/libs/spark-streaming-kafka_2.10-1.6.0.jar
#packages: org.apache.spark:spark-streaming-kafka:1.6.0
[local-singlecore]
master: local[1]
[docker]
master: mesos://mesos-master-000-%(ENV)s.mistsys.net:7077
deps_dir: /data/_dependencies
jars: %(deps_dir)s/libs/aws-java-sdk-1.7.4.jar,%(deps_dir)s/libs/hadoop-aws-2.7.0.jar,%(deps_dir)s/libs/spark-cassandra-connector_2.10-1.6.0-M1.jar,%(deps_dir)s/libs/spark-streaming-kafka_2.10-1.6.0.jar
tmp_dir: /src/tmp
# this is very very specific to Mesos/Chronos
projects_dir: /mnt/mesos/sandbox/spark-dev
enable_event_logs: true
# should be inside docker
mesos_libs_dir: /usr/local/lib
metrics_conf: /conf/metrics.properties
remote_deps_dir: /data/_dependencies
[vagrant-submitter]
master: mesos://zk://10.10.4.2:2181/mesos
# You want your executors to find the files, don't you?
remote_deps_dir: /vagrant/_dependencies
mesos_libs_dir: _dependencies/mesos-0.25.0/build/src/.libs
[qubole-example]
deployment_environment: qubole
master: https://api.qubole.com/api/v1.2
# custom remote services are prefixed with <name>- they're only looked at
# if the deployment_environment matches the prefix
# To be filled by environment variables
qubole-auth_token: %(QUBOLE_API_TOKEN)s
qubole-cluster_id: 1111
# This needs to be changed at some point
qubole-py_files: /tmp/mist.zip
qubole-cluster_label: mist-us-east-1-spark
qubole-update_dependencies_script: s3://path/to/script/update_files.sh
qubole-config_path: /mist/spark-dev/config/config_staging.json
[databricks-example]
deployment_environment: databricks
master: https://community.cloud.databricks.com/api/1.2
databricks-cluster_id: 0817-031357-belle745
databricks-username : %(DATABRICKS_USERNAME)s
databricks-password: %(DATABRICKS_PASSWORD)s
[vagrant]
master: mesos://zk://10.10.4.2:2181/mesos
# You want your executors to find the files, don't you?
remote_deps_dir: /vagrant/_dependencies
mesos_libs_dir: /usr/local/lib