fix(su): seperate ARWEAVE_URL and GRAPHQL_URL, and fix SU_DATA_DIR to…

… be optional #891 #878

servers/su/.env.example

@@ -1,7 +1,12 @@
-SU_WALLET_PATH=./.wallet.json
-DATABASE_URL=postgresql://user:password@localhost/su
-GATEWAY_URL=https://arweave.net/
+SU_WALLET_PATH=wallet.json
+DATABASE_URL=postgresql://postgres:password@localhost/su
+GRAPHQL_URL=https://arweave-search.goldsky.com
+ARWEAVE_URL=https://arweave.net
 UPLOAD_NODE_URL=https://up.arweave.net
 MODE=su
-SCHEDULER_LIST_PATH=""
-PORT=9000
+SCHEDULER_LIST_PATH=schedulers.json
+SU_DATA_DIR=/home/rocksdb
+USE_DISK=true
+MIGRATION_BATCH_SIZE=10
+DB_WRITE_CONNECTIONS=1
+DB_READ_CONNECTIONS=40

servers/su/README.md

@@ -36,7 +36,9 @@ Create a .env file with the following variables, or set them in the OS:
 - `SU_WALLET_PATH` a local filepath to an arweave wallet the SU will use to write tx's
 - `DATABASE_URL` a postgres database url, you must have a postgres database called `su`
 - `DATABASE_READ_URL` an optional separate postgres database url for reads
-- `GATEWAY_URL`an arweave gateway url to write to `https://arweave.net/`
+- `GRAPHQL_URL`an url for the arweave graphql interface `https://arweave-search.goldsky.com`
+- `ARWEAVE_URL`an arweave gateway url to fetch actual transactions and network info from `https://arweave.net/`
+- `GATEWAY_URL`an default fallback for the above 2. Must provide graphql, network info, and tx fetching.
 - `UPLOAD_NODE_URL` an uploader url such as `https://up.arweave.net`
 - `MODE` can be either value `su` or `router` but for local development use `su`
 - `SCHEDULER_LIST_PATH` a list of schedulers only used for `router` MODE. Ignore when in `su` MODE, just set it to `""`.
@@ -106,11 +108,14 @@ environment variables. You will also need to make sure the database url is acces
 in the container. 
 
 - `SU_WALLET_PATH` a local filepath to an arweave wallet the SU will use to write tx's
-- `DATABASE_URL` a postgres database url, you must create a postgres database called `su`
-- `GATEWAY_URL`an arweave gateway url to write to `https://arweave.net/`
+- `DATABASE_URL` a postgres database url, you must have a postgres database called `su`
+- `DATABASE_READ_URL` an optional separate postgres database url for reads
+- `GRAPHQL_URL`an url for the arweave graphql interface `https://arweave-search.goldsky.com`
+- `ARWEAVE_URL`an arweave gateway url to fetch actual transactions and network info from `https://arweave.net/`
+- `GATEWAY_URL`an default fallback for the above 2. Must provide graphql, network info, and tx fetching.
 - `UPLOAD_NODE_URL` an uploader url such as `https://up.arweave.net`
 - `MODE` can be either value `su` or `router` but for local development use `su`
-- `SCHEDULER_LIST_PATH` a list of schedulers only used for `router` MODE. Ignore in `su` mode just set it to `""`.
+- `SCHEDULER_LIST_PATH` a list of schedulers only used for `router` MODE. Ignore when in `su` MODE, just set it to `""`.
 - `DB_WRITE_CONNECTIONS` how many db connections in the writer pool,defaults to 10
 - `DB_READ_CONNECTIONS` how many db connections in the reader pool, default to 10
 - `USE_DISK` whether or not to write and read rocksdb, this is a performance enhancement for the data storage layer

servers/su/src/domain/clients/gateway.rs

@@ -88,8 +88,8 @@ impl ArweaveGateway {
 
     async fn network_info_fetch() -> Result<NetworkInfo, String> {
         let config = AoConfig::new(Some("su".to_string())).expect("Failed to read configuration");
-        let gateway_url = config.gateway_url;
-        let url = Url::parse(&gateway_url).map_err(|e| format!("{:?}", e))?;
+        let arweave_url = config.arweave_url;
+        let url = Url::parse(&arweave_url).map_err(|e| format!("{:?}", e))?;
 
         let network_client = NetworkInfoClient::new(url);
 
@@ -131,9 +131,9 @@ impl ArweaveGateway {
 impl Gateway for ArweaveGateway {
     async fn check_head(&self, tx_id: String) -> Result<bool, String> {
         let config = AoConfig::new(Some("su".to_string())).expect("Failed to read configuration");
-        let gateway_url = config.gateway_url;
+        let arweave_url = config.arweave_url;
 
-        let url = match Url::parse(&gateway_url) {
+        let url = match Url::parse(&arweave_url) {
             Ok(u) => u,
             Err(e) => return Err(format!("{}", e)),
         };
@@ -166,9 +166,9 @@ impl Gateway for ArweaveGateway {
 
     async fn status(&self, tx_id: &String) -> Result<TxStatus, String> {
         let config = AoConfig::new(Some("su".to_string())).expect("Failed to read configuration");
-        let gateway_url = config.gateway_url;
+        let arweave_url = config.arweave_url;
 
-        let url = match Url::parse(&gateway_url) {
+        let url = match Url::parse(&arweave_url) {
             Ok(u) => u,
             Err(e) => return Err(format!("{}", e)),
         };
@@ -205,7 +205,7 @@ impl Gateway for ArweaveGateway {
 
     async fn gql_tx(&self, tx_id: &String) -> Result<GatewayTx, String> {
         let config = AoConfig::new(Some("su".to_string())).expect("Failed to read configuration");
-        let gateway_url = config.gateway_url;
+        let graphql_url = config.graphql_url;
         let client = Client::new();
 
         let query = serde_json::json!({
@@ -220,7 +220,7 @@ impl Gateway for ArweaveGateway {
             .map_err(|e| GatewayErrorType::GraphQLError(e.to_string()))?;
 
         let response = client
-            .post(format!("{}/graphql", gateway_url))
+            .post(format!("{}/graphql", graphql_url))
             .header("Content-Type", "application/json")
             .body(query_string)
             .send()

servers/su/src/domain/config.rs

@@ -9,7 +9,8 @@ pub struct AoConfig {
     pub database_url: String,
     pub database_read_url: String,
     pub su_wallet_path: String,
-    pub gateway_url: String,
+    pub graphql_url: String,
+    pub arweave_url: String,
     pub upload_node_url: String,
     pub mode: String,
     pub scheduler_list_path: String,
@@ -35,6 +36,10 @@ impl AoConfig {
             Ok(val) => val == "true",
             Err(_e) => false,
         };
+        let su_data_dir = match use_disk {
+          true => env::var("SU_DATA_DIR")?,
+          false => "".to_string()
+        };
         let migration_batch_size = match env::var("MIGRATION_BATCH_SIZE") {
             Ok(val) => val.parse().unwrap(),
             Err(_e) => 1000,
@@ -47,16 +52,25 @@ impl AoConfig {
             Ok(val) => val.parse().unwrap(),
             Err(_e) => 10,
         };
+        let graphql_url = match env::var("GRAPHQL_URL") {
+          Ok(val) => val,
+          Err(_e) => env::var("GATEWAY_URL")?
+        };
+        let arweave_url = match env::var("ARWEAVE_URL") {
+          Ok(val) => val,
+          Err(_e) => env::var("GATEWAY_URL")?
+        };
         Ok(AoConfig {
             database_url: env::var("DATABASE_URL")?,
             database_read_url,
             su_wallet_path: env::var("SU_WALLET_PATH")?,
-            gateway_url: env::var("GATEWAY_URL")?,
+            graphql_url,
+            arweave_url,
             upload_node_url: env::var("UPLOAD_NODE_URL")?,
             mode: mode_out,
             scheduler_list_path: env::var("SCHEDULER_LIST_PATH")?,
             use_disk,
-            su_data_dir: env::var("SU_DATA_DIR")?,
+            su_data_dir,
             migration_batch_size,
             db_write_connections,
             db_read_connections,
@@ -65,15 +79,6 @@ impl AoConfig {
 }
 
 impl Config for AoConfig {
-    fn su_wallet_path(&self) -> String {
-        self.su_wallet_path.clone()
-    }
-    fn upload_node_url(&self) -> String {
-        self.upload_node_url.clone()
-    }
-    fn gateway_url(&self) -> String {
-        self.gateway_url.clone()
-    }
     fn mode(&self) -> String {
         self.mode.clone()
     }

servers/su/src/domain/core/dal.rs

@@ -58,9 +58,6 @@ pub trait ScheduleProvider {
 }
 
 pub trait Config: Send + Sync {
-    fn su_wallet_path(&self) -> String;
-    fn upload_node_url(&self) -> String;
-    fn gateway_url(&self) -> String;
     fn mode(&self) -> String;
     fn scheduler_list_path(&self) -> String;
 }