[docs, core] Rename module, add more profiling data and other minor f…

…ormatting changes (#11)
microsoft · May 14, 2024 · e26eb8f · e26eb8f
1 parent bee57da
commit e26eb8f
Show file tree

Hide file tree

Showing 183 changed files with 374,220 additions and 480 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,5 @@
+{
+    "cSpell.words": [
+        "vidur"
+    ]
+}
diff --git a/Makefile b/Makefile
@@ -2,23 +2,23 @@
 .DEFAULT_GOAL := help
 
 lint/flake8: ## check style with flake8
-	flake8 simulator
+	flake8 vidur
 
 lint/black: ## check style with black
-	black --check simulator
+	black --check vidur
 
 lint/isort: ## check style with isort
-	isort --check-only --profile black simulator
+	isort --check-only --profile black vidur
 
 lint: lint/black lint/isort ## check style
 
 format/black: ## format code with black
-	black simulator
+	black vidur
 
 format/autopep8: ## format code with autopep8
-	autopep8 --in-place --recursive simulator/
+	autopep8 --in-place --recursive vidur/
 
 format/isort: ## format code with isort
-	isort --profile black simulator
+	isort --profile black vidur
 
 format: format/isort format/black ## format code
diff --git a/README.md b/README.md
@@ -48,20 +48,20 @@ wandb login --host https://<your-org>.wandb.io
 To opt out of wandb, pick any one of the following methods:
 
 1. `export WANDB_MODE=disabled` in your shell or add this in `~/.zshrc` or `~/.bashrc`. Remember to reload using `source ~/.zshrc`.
-2. Set `wandb_project` and `wandb_group` as `""` in `simulator/config/default.yml`. Also, remove these CLI params from the shell command with which the simulator is invoked.
+2. Set `wandb_project` and `wandb_group` as `""` in `vidur/config/default.yml`. Also, remove these CLI params from the shell command with which the simulator is invoked.
 
 ## Running the simulator
 
 To run the simulator, execute the following command from the repository root,
 
 ```sh
-python -m simulator.main
+python -m vidur.main
 ```
 
 or a big example with all the parameters,
 
 ```sh
-python -m simulator.main  \
+python -m vidur.main  \
 --replica_device a100 \
 --replica_model_name meta-llama/Llama-2-7b-hf  \
 --cluster_num_replicas 1 \
@@ -84,13 +84,13 @@ python -m simulator.main  \
 --vllm_scheduler_max_tokens_in_batch 4096
 ```
 
-The simulator supports a plethora of parameters for the simulation description which can be found [here](simulator/config/README.md).
+The simulator supports a plethora of parameters for the simulation description which can be found [here](vidur/config/README.md).
 
-The metrics will be logged to wandb directly and a copy will be stored in the `simulator_output` directory along with the chrome trace. A description of all the logged metrics can be found [here](simulator/metrics/README.md).
+The metrics will be logged to wandb directly and a copy will be stored in the `simulator_output` directory along with the chrome trace. A description of all the logged metrics can be found [here](vidur/metrics/README.md).
 
 ## Adding a new model
 
-Instructions on adding a new model can be found [here](simulator/profiling/README.md).
+Instructions on adding a new model can be found [here](vidur/profiling/README.md).
 
 ## Formatting Code
 

diff --git a/data/profiling/compute/a40/Qwen/Qwen-72B/attention.csv b/data/profiling/compute/a40/Qwen/Qwen-72B/attention.csv
diff --git a/data/profiling/compute/a40/Qwen/Qwen-72B/mlp.csv b/data/profiling/compute/a40/Qwen/Qwen-72B/mlp.csv
diff --git a/data/profiling/compute/a40/codellama/CodeLlama-34b-Instruct-hf/attention.csv b/data/profiling/compute/a40/codellama/CodeLlama-34b-Instruct-hf/attention.csv
diff --git a/data/profiling/compute/a40/codellama/CodeLlama-34b-Instruct-hf/mlp.csv b/data/profiling/compute/a40/codellama/CodeLlama-34b-Instruct-hf/mlp.csv
diff --git a/data/profiling/compute/a40/internlm/internlm-20b/attention.csv b/data/profiling/compute/a40/internlm/internlm-20b/attention.csv
diff --git a/data/profiling/compute/a40/internlm/internlm-20b/mlp.csv b/data/profiling/compute/a40/internlm/internlm-20b/mlp.csv
diff --git a/data/profiling/compute/a40/meta-llama/Llama-2-70b-hf/attention.csv b/data/profiling/compute/a40/meta-llama/Llama-2-70b-hf/attention.csv
diff --git a/data/profiling/compute/a40/meta-llama/Llama-2-70b-hf/mlp.csv b/data/profiling/compute/a40/meta-llama/Llama-2-70b-hf/mlp.csv
diff --git a/data/profiling/compute/a40/meta-llama/Llama-2-7b-hf/attention.csv b/data/profiling/compute/a40/meta-llama/Llama-2-7b-hf/attention.csv
diff --git a/data/profiling/compute/a40/meta-llama/Llama-2-7b-hf/mlp.csv b/data/profiling/compute/a40/meta-llama/Llama-2-7b-hf/mlp.csv
diff --git a/data/profiling/compute/a40/microsoft/phi-2/attention.csv b/data/profiling/compute/a40/microsoft/phi-2/attention.csv
diff --git a/data/profiling/compute/a40/microsoft/phi-2/mlp.csv b/data/profiling/compute/a40/microsoft/phi-2/mlp.csv
diff --git a/data/profiling/compute/h100/Qwen/Qwen-72B/mlp.csv b/data/profiling/compute/h100/Qwen/Qwen-72B/mlp.csv
diff --git a/data/profiling/compute/h100/codellama/CodeLlama-34b-Instruct-hf/mlp.csv b/data/profiling/compute/h100/codellama/CodeLlama-34b-Instruct-hf/mlp.csv
diff --git a/data/profiling/compute/h100/internlm/internlm-20b/mlp.csv b/data/profiling/compute/h100/internlm/internlm-20b/mlp.csv
diff --git a/data/profiling/compute/h100/meta-llama/Llama-2-70b-hf/mlp.csv b/data/profiling/compute/h100/meta-llama/Llama-2-70b-hf/mlp.csv
diff --git a/data/profiling/compute/h100/meta-llama/Llama-2-7b-hf/mlp.csv b/data/profiling/compute/h100/meta-llama/Llama-2-7b-hf/mlp.csv
diff --git a/data/profiling/compute/h100/microsoft/phi-2/mlp.csv b/data/profiling/compute/h100/microsoft/phi-2/mlp.csv
diff --git a/data/profiling/network/a100_dgx/all_reduce.csv b/data/profiling/network/a100_dgx/all_reduce.csv
diff --git a/data/profiling/network/a100_dgx/send_recv.csv b/data/profiling/network/a100_dgx/send_recv.csv
diff --git a/...filing/network/h100_nvlink/all_reduce.csv → ...profiling/network/h100_dgx/all_reduce.csv b/...filing/network/h100_nvlink/all_reduce.csv → ...profiling/network/h100_dgx/all_reduce.csv
diff --git a/data/profiling/network/h100_dgx/send_recv.csv b/data/profiling/network/h100_dgx/send_recv.csv
diff --git a/setup.py b/setup.py
@@ -6,8 +6,8 @@
     python_requires='>=3.10',
     description="A LLM inference cluster simulator",
     include_package_data=True,
-    keywords='simulator',
-    name='simulator',
-    packages=find_packages(include=['simulator', 'simulator.*']),
+    keywords='vidur',
+    name='vidur',
+    packages=find_packages(include=['vidur', 'vidur.*']),
     version='0.0.1',
 )
diff --git a/simulator/config/__init__.py b/simulator/config/__init__.py
diff --git a/simulator/entities/__init__.py b/simulator/entities/__init__.py
diff --git a/simulator/events/__init__.py b/simulator/events/__init__.py
diff --git a/simulator/execution_time_predictor/__init__.py b/simulator/execution_time_predictor/__init__.py
diff --git a/simulator/main.py b/simulator/main.py
diff --git a/simulator/metrics/__init__.py b/simulator/metrics/__init__.py
diff --git a/simulator/request_generator/__init__.py b/simulator/request_generator/__init__.py
diff --git a/simulator/scheduler/global_scheduler/__init__.py b/simulator/scheduler/global_scheduler/__init__.py
diff --git a/simulator/scheduler/replica_stage_scheduler/__init__.py b/simulator/scheduler/replica_stage_scheduler/__init__.py
diff --git a/simulator/types/__init__.py b/simulator/types/__init__.py
diff --git a/simulator/config/README.md → vidur/config/README.md b/simulator/config/README.md → vidur/config/README.md
@@ -1,6 +1,6 @@
 # Understanding the parameters taken by the simulator
 
-The [default.yml](simulator/config/default.yml) is the comprehensive list of all parameters taken by the simulator. While invoking the simulator, any of these parameters can be overrided. Running only `python -m simulator.main` means that all the parameters are taken from the `default.yml` file and no overrides.
+The [default.yml](vidur/config/default.yml) is the comprehensive list of all parameters taken by the simulator. While invoking the simulator, any of these parameters can be overrided. Running only `python -m vidur.main` means that all the parameters are taken from the `default.yml` file and no overrides.
 The parameters descriptions are given below:
 
 1. `seed`: Random seed which is set in multiple random generators notably the request length and inter-request time generators. This is useful for reproducibility.
@@ -34,7 +34,7 @@ All the output files corresponding to the invocation are stored under this direc
         1. `fp16_tflops`: TFLOPS of the GPU in FP16. This is used to predict the execution time of the model.
         2. `total_memory_gb`: Total memory of the GPU in GB. This is used in memory calculations of the model weights, KV cache etc.
         3. For `a100`: `fp16_tflops: 312`, `total_memory_gb: 80`
-10. `request_generator`: The simulator contains a comprehensive request generator. See [here](simulator/request_generator)
+10. `request_generator`: The simulator contains a comprehensive request generator. See [here](vidur/request_generator)
     1. `provider`: The request generator to use. Currently supported are `synthetic`, `trace`. `synthetic` generates requests from a synthetic distribution. `trace` generates requests from a real-world trace.
     2. `max_tokens`: Maximum number of tokens in a request. Requests generated from the trace are capped / clipped at this number. `P:D ratio` is preserved in case of clipping.
 11. `synthetic_request_generator`: This section is used to further define the synthetic request generator. Only required if `request_generator_provider` is set to `synthetic`.
@@ -87,9 +87,9 @@ All the output files corresponding to the invocation are stored under this direc
 22. `simulator`:
     1. `time_limit`: Time limit for the simulator to run. This is useful to run the simulator for a fixed amount of time. The simulator will stop after this time limit is reached. Default is no limit. TODO: Verify the functionality of this parameter.
 23. `global_scheduler`: This is the scheduler which determines which replica to send the request to.
-    1. `provider`: `round_robin`, `random`, `lor`. See [here](simulator/schedulers/global_schedulers) for more details.
+    1. `provider`: `round_robin`, `random`, `lor`. See [here](vidur/schedulers/global_schedulers) for more details.
 24. `replica_scheduler`: This is the scheduler which determines how to schedule the requests on a replica.
-    1. `provider`: `orca`, `sarathi`, and `vllm`. See [here](simulator/schedulers/replica_schedulers) for more details.
+    1. `provider`: `orca`, `sarathi`, and `vllm`. See [here](vidur/schedulers/replica_schedulers) for more details.
     2. `batch_size_cap`: Maximum permissible batch size. Set carefully for `orca`. Have a high limit for other schedulers. They will auto-adjust.
     3. `num_blocks`: TODO. Ignore this parameter for now.
 25. `orca_scheduler`: Only required if `replica_scheduler_provider` is set to `orca`.

diff --git a/vidur/config/__init__.py b/vidur/config/__init__.py
@@ -0,0 +1,3 @@
+from vidur.config.config import Config
+
+__all__ = [Config]
diff --git a/simulator/config/config.py → vidur/config/config.py b/simulator/config/config.py → vidur/config/config.py
@@ -1,13 +1,13 @@
 import argparse
 import datetime
-import logging
 import os
 
 import yaml
 
-from simulator.constants import DEFAULT_CONFIG_FILE, DEVICE_CONFIG_DIR, MODEL_CONFIG_DIR
+from vidur.constants import DEFAULT_CONFIG_FILE, DEVICE_CONFIG_DIR, MODEL_CONFIG_DIR
+from vidur.logger import init_logger
 
-logger = logging.getLogger(__name__)
+logger = init_logger(__name__)
 
 
 class Config:

diff --git a/simulator/config/default.yml → vidur/config/default.yml b/simulator/config/default.yml → vidur/config/default.yml
@@ -38,7 +38,7 @@ trace_request_generator:
 
 # Config for synthetic trace generator
 trace_request_length_generator:
-  trace_file: ./data/processed_traces/lmsys_chat_1m_conversation_stats_llama2_tokenizer.csv
+  trace_file: ./data/processed_traces/arxiv_summarization_stats_llama2_tokenizer_filtered_v2.csv
   prefill_scale_factor: 1
   decode_scale_factor:  1
 

diff --git a/simulator/config_optimizer/__init__.py → vidur/config_optimizer/__init__.py b/simulator/config_optimizer/__init__.py → vidur/config_optimizer/__init__.py
diff --git a/...tor/config_optimizer/analyzer/__init__.py → vidur/config_optimizer/analyzer/__init__.py b/...tor/config_optimizer/analyzer/__init__.py → vidur/config_optimizer/analyzer/__init__.py
diff --git a/...optimizer/analyzer/bottleneck_analyzer.py → ...optimizer/analyzer/bottleneck_analyzer.py b/...optimizer/analyzer/bottleneck_analyzer.py → ...optimizer/analyzer/bottleneck_analyzer.py
@@ -1,4 +1,4 @@
-from simulator.config_optimizer.analyzer.bottleneck_cases import *
+from vidur.config_optimizer.analyzer.bottleneck_cases import *
 
 
 class BottleneckAnalyzer:

diff --git a/...ig_optimizer/analyzer/bottleneck_cases.py → ...ig_optimizer/analyzer/bottleneck_cases.py b/...ig_optimizer/analyzer/bottleneck_cases.py → ...ig_optimizer/analyzer/bottleneck_cases.py
diff --git a/...or/config_optimizer/analyzer/constants.py → vidur/config_optimizer/analyzer/constants.py b/...or/config_optimizer/analyzer/constants.py → vidur/config_optimizer/analyzer/constants.py
diff --git a/..._optimizer/analyzer/dashboard/__init__.py → ..._optimizer/analyzer/dashboard/__init__.py b/..._optimizer/analyzer/dashboard/__init__.py → ..._optimizer/analyzer/dashboard/__init__.py
diff --git a/...zer/analyzer/dashboard/assets/favicon.png → ...zer/analyzer/dashboard/assets/favicon.png b/...zer/analyzer/dashboard/assets/favicon.png → ...zer/analyzer/dashboard/assets/favicon.png
diff --git a/...izer/analyzer/dashboard/assets/index.html → ...izer/analyzer/dashboard/assets/index.html b/...izer/analyzer/dashboard/assets/index.html → ...izer/analyzer/dashboard/assets/index.html
diff --git a/...izer/analyzer/dashboard/assets/nginx.conf → ...izer/analyzer/dashboard/assets/nginx.conf b/...izer/analyzer/dashboard/assets/nginx.conf → ...izer/analyzer/dashboard/assets/nginx.conf
diff --git a/...er/analyzer/dashboard/best_config_page.py → ...er/analyzer/dashboard/best_config_page.py b/...er/analyzer/dashboard/best_config_page.py → ...er/analyzer/dashboard/best_config_page.py
@@ -2,12 +2,9 @@
 import plotly.graph_objs as go
 import streamlit as st
 
-from simulator.config_optimizer.analyzer.bottleneck_analyzer import BottleneckAnalyzer
-from simulator.config_optimizer.analyzer.constants import (
-    AXIS_COLS,
-    AXIS_COLS_LONG_TO_SHORT,
-)
-from simulator.config_optimizer.analyzer.dashboard.utils import (
+from vidur.config_optimizer.analyzer.bottleneck_analyzer import BottleneckAnalyzer
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS, AXIS_COLS_LONG_TO_SHORT
+from vidur.config_optimizer.analyzer.dashboard.utils import (
     add_advanced_filters,
     add_small_divider,
     convert_config_row_to_comparison_point,

diff --git a/...analyzer/dashboard/config_compare_page.py → ...analyzer/dashboard/config_compare_page.py b/...analyzer/dashboard/config_compare_page.py → ...analyzer/dashboard/config_compare_page.py
@@ -3,11 +3,8 @@
 import pandas as pd
 import streamlit as st
 
-from simulator.config_optimizer.analyzer.constants import AXIS_COLS
-from simulator.config_optimizer.analyzer.dashboard.utils import (
-    add_small_divider,
-    plot_cdf,
-)
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS
+from vidur.config_optimizer.analyzer.dashboard.utils import add_small_divider, plot_cdf
 
 
 def render_comparison_points(df):

diff --git a/.../analyzer/dashboard/cost_analysis_page.py → .../analyzer/dashboard/cost_analysis_page.py b/.../analyzer/dashboard/cost_analysis_page.py → .../analyzer/dashboard/cost_analysis_page.py
@@ -1,9 +1,9 @@
 import plotly.express as px
 import streamlit as st
 
-from simulator.config_optimizer.analyzer.bottleneck_analyzer import BottleneckAnalyzer
-from simulator.config_optimizer.analyzer.constants import AXIS_COLS
-from simulator.config_optimizer.analyzer.dashboard.utils import (
+from vidur.config_optimizer.analyzer.bottleneck_analyzer import BottleneckAnalyzer
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS
+from vidur.config_optimizer.analyzer.dashboard.utils import (
     add_advanced_filters,
     add_model_trace_selector,
     add_small_divider,

diff --git a/...ptimizer/analyzer/dashboard/intro_page.py → ...ptimizer/analyzer/dashboard/intro_page.py b/...ptimizer/analyzer/dashboard/intro_page.py → ...ptimizer/analyzer/dashboard/intro_page.py
@@ -1,6 +1,6 @@
 import streamlit as st
 
-from simulator.config_optimizer.analyzer.dashboard.utils import add_small_divider
+from vidur.config_optimizer.analyzer.dashboard.utils import add_small_divider
 
 
 def render_intro_page():

diff --git a/...nfig_optimizer/analyzer/dashboard/main.py → ...nfig_optimizer/analyzer/dashboard/main.py b/...nfig_optimizer/analyzer/dashboard/main.py → ...nfig_optimizer/analyzer/dashboard/main.py
@@ -6,28 +6,28 @@
 
 st.set_page_config(page_title="Vidur Demo", layout="wide")
 
-from simulator.config_optimizer.analyzer.constants import AXIS_COLS, PRETTY_NAMES
-from simulator.config_optimizer.analyzer.dashboard.best_config_page import (
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS, PRETTY_NAMES
+from vidur.config_optimizer.analyzer.dashboard.best_config_page import (
     render_best_config_selection_page,
 )
-from simulator.config_optimizer.analyzer.dashboard.config_compare_page import (
+from vidur.config_optimizer.analyzer.dashboard.config_compare_page import (
     render_config_comparison_page,
 )
-from simulator.config_optimizer.analyzer.dashboard.cost_analysis_page import (
+from vidur.config_optimizer.analyzer.dashboard.cost_analysis_page import (
     render_cost_analysis_page,
 )
-from simulator.config_optimizer.analyzer.dashboard.intro_page import render_intro_page
-from simulator.config_optimizer.analyzer.dashboard.pareto_curve_page import (
+from vidur.config_optimizer.analyzer.dashboard.intro_page import render_intro_page
+from vidur.config_optimizer.analyzer.dashboard.pareto_curve_page import (
     render_pareto_curve_page,
 )
-from simulator.config_optimizer.analyzer.dashboard.search_analysis_page import (
+from vidur.config_optimizer.analyzer.dashboard.search_analysis_page import (
     render_search_analysis_page,
 )
-from simulator.config_optimizer.analyzer.dashboard.utils import (
+from vidur.config_optimizer.analyzer.dashboard.utils import (
     get_config_name,
     hide_anchor_link,
 )
-from simulator.config_optimizer.analyzer.utils import get_trace_name
+from vidur.config_optimizer.analyzer.utils import get_trace_name
 
 
 def select_best_run(

diff --git a/...r/analyzer/dashboard/pareto_curve_page.py → ...r/analyzer/dashboard/pareto_curve_page.py b/...r/analyzer/dashboard/pareto_curve_page.py → ...r/analyzer/dashboard/pareto_curve_page.py
@@ -4,9 +4,9 @@
 import streamlit as st
 from paretoset import paretoset
 
-from simulator.config_optimizer.analyzer.bottleneck_analyzer import BottleneckAnalyzer
-from simulator.config_optimizer.analyzer.constants import AXIS_COLS
-from simulator.config_optimizer.analyzer.dashboard.utils import (
+from vidur.config_optimizer.analyzer.bottleneck_analyzer import BottleneckAnalyzer
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS
+from vidur.config_optimizer.analyzer.dashboard.utils import (
     add_advanced_filters,
     add_model_trace_selector,
     add_small_divider,

diff --git a/...nalyzer/dashboard/search_analysis_page.py → ...nalyzer/dashboard/search_analysis_page.py b/...nalyzer/dashboard/search_analysis_page.py → ...nalyzer/dashboard/search_analysis_page.py
@@ -1,10 +1,7 @@
 import streamlit as st
 
-from simulator.config_optimizer.analyzer.constants import AXIS_COLS
-from simulator.config_optimizer.analyzer.dashboard.utils import (
-    add_small_divider,
-    plot_cdf,
-)
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS
+from vidur.config_optimizer.analyzer.dashboard.utils import add_small_divider, plot_cdf
 
 
 def render_search_instances(df):

diff --git a/...fig_optimizer/analyzer/dashboard/utils.py → ...fig_optimizer/analyzer/dashboard/utils.py b/...fig_optimizer/analyzer/dashboard/utils.py → ...fig_optimizer/analyzer/dashboard/utils.py
@@ -2,7 +2,7 @@
 import randomname
 import streamlit as st
 
-from simulator.config_optimizer.analyzer.constants import AXIS_COLS, PRETTY_NAMES
+from vidur.config_optimizer.analyzer.constants import AXIS_COLS, PRETTY_NAMES
 
 
 def hide_anchor_link():

diff --git a/...imizer/analyzer/generate_pareto_curves.py → ...imizer/analyzer/generate_pareto_curves.py b/...imizer/analyzer/generate_pareto_curves.py → ...imizer/analyzer/generate_pareto_curves.py
@@ -9,13 +9,13 @@
 from matplotlib.colors import LinearSegmentedColormap
 from paretoset import paretoset
 
-from simulator.config_optimizer.analyzer.constants import (
+from vidur.config_optimizer.analyzer.constants import (
     AXIS_COLS,
     CONFIG_KEY,
     METRICS,
     PRETTY_NAMES,
 )
-from simulator.config_optimizer.analyzer.utils import get_trace_name
+from vidur.config_optimizer.analyzer.utils import get_trace_name
 
 
 def plot_pareto_curve_under_slos(