Skip to content

Commit

Permalink
feat(dl): support new safetensors model & drop gitcode 👎
Browse files Browse the repository at this point in the history
  • Loading branch information
@fumiama
fumiama committed Aug 28, 2024
1 parent 9f9abec commit c06f1d4
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 47 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/checksum.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:

- name: Run RVC-Models-Downloader
run: |
wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.7/rvcmd_linux_amd64.deb
wget https://github.com/fumiama/RVC-Models-Downloader/releases/download/v0.2.8/rvcmd_linux_amd64.deb
sudo apt -y install ./rvcmd_linux_amd64.deb
rm -f ./rvcmd_linux_amd64.deb
rvcmd -notrs -w 1 -notui assets/chtts
Expand Down
9 changes: 6 additions & 3 deletions ChatTTS/res/sha256_map.json
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
{
"sha256_asset_Decoder_pt" : "9964e36e840f0e3a748c5f716fe6de6490d2135a5f5155f4a642d51860e2ec38",
"sha256_asset_Decoder_pt" : "9964e36e840f0e3a748c5f716fe6de6490d2135a5f5155f4a642d51860e2ec38",
"sha256_asset_DVAE_full_pt" : "553eb75763511e23f3e5f86303e2163c5ca775489d637fb635d979c8ae58bbe5",
"sha256_asset_GPT_pt" : "d7d4ee6461ea097a2be23eb40d73fb94ad3b3d39cb64fbb50cb3357fd466cadb",
"sha256_asset_Vocos_pt" : "09a670eda1c08b740013679c7a90ebb7f1a97646ea7673069a6838e6b51d6c58",
"sha256_asset_Embed_safetensors" : "2ff0be7134934155741b643b74e32fb6bf3eec41257984459b2ed60cdb4c48b0",
"sha256_asset_Vocos_pt" : "09a670eda1c08b740013679c7a90ebb7f1a97646ea7673069a6838e6b51d6c58",

"sha256_asset_gpt_config_json" : "0aaa1ecd96c49ad4f473459eb1982fa7ad79fa5de08cde2781bf6ad1f9a0c236",
"sha256_asset_gpt_model_safetensors" : "cd0806fd971f52f6a22c923ec64982b305e817bcc41ca83417fcf9141b984a0f",

"sha256_asset_tokenizer_special_tokens_map_json": "bd0ac9d9bb1657996b5c5fbcaa7d80f8de530d01a283da97f89deae5b1b8d011",
"sha256_asset_tokenizer_tokenizer_config_json" : "43e9d658b554fa5ee8d8e1d763349323bfef1ed7a89c0794220ab8861387d421",
Expand Down
79 changes: 40 additions & 39 deletions ChatTTS/utils/dl.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import hashlib
import requests
from io import BytesIO
from typing import Dict
from typing import Dict, Tuple, Optional
from mmap import mmap, ACCESS_READ

from .log import logger
Expand Down Expand Up @@ -42,46 +42,54 @@ def check_model(
os.remove(bakfile)
return True

def check_folder(base_dir: Path, *innder_dirs: str, names: Tuple[str], sha256_map: Dict[str, str], update=False) -> bool:
key = "sha256_"
current_dir = base_dir
for d in innder_dirs:
current_dir /= d
key += f"{d}_"

for model in names:
menv = model.replace(".", "_")
if not check_model(
current_dir, model, sha256_map[f"{key}{menv}"], update
):
return False
return True

def check_all_assets(base_dir: Path, sha256_map: Dict[str, str], update=False) -> bool:
logger.get_logger().info("checking assets...")

current_dir = base_dir / "asset"
names = [
if not check_folder(base_dir, "asset", names=(
"Decoder.pt",
"DVAE_full.pt",
"GPT.pt",
"Embed.safetensors",
"Vocos.pt",
]
for model in names:
menv = model.replace(".", "_")
if not check_model(
current_dir, model, sha256_map[f"sha256_asset_{menv}"], update
):
return False
), sha256_map=sha256_map, update=update):
return False

if not check_folder(base_dir, "asset", "gpt", names=(
"config.json",
"model.safetensors",
), sha256_map=sha256_map, update=update):
return False

current_dir = base_dir / "asset" / "tokenizer"
names = [
if not check_folder(base_dir, "asset", "tokenizer", names=(
"special_tokens_map.json",
"tokenizer_config.json",
"tokenizer.json",
]
for model in names:
menv = model.replace(".", "_")
if not check_model(
current_dir, model, sha256_map[f"sha256_asset_tokenizer_{menv}"], update
):
return False
), sha256_map=sha256_map, update=update):
return False

logger.get_logger().info("all assets are already latest.")
return True


def download_and_extract_tar_gz(url: str, folder: str):
def download_and_extract_tar_gz(url: str, folder: str, headers: Optional[Dict[str, str]] = None):
import tarfile

logger.get_logger().info(f"downloading {url}")
response = requests.get(url, stream=True, timeout=(5, 10))
response = requests.get(url, headers=headers, stream=True, timeout=(10, 3))
with BytesIO() as out_file:
out_file.write(response.content)
out_file.seek(0)
Expand All @@ -91,11 +99,11 @@ def download_and_extract_tar_gz(url: str, folder: str):
logger.get_logger().info(f"extracted into {folder}")


def download_and_extract_zip(url: str, folder: str):
def download_and_extract_zip(url: str, folder: str, headers: Optional[Dict[str, str]] = None):
import zipfile

logger.get_logger().info(f"downloading {url}")
response = requests.get(url, stream=True, timeout=(5, 10))
response = requests.get(url, headers=headers, stream=True, timeout=(10, 3))
with BytesIO() as out_file:
out_file.write(response.content)
out_file.seek(0)
Expand All @@ -105,15 +113,15 @@ def download_and_extract_zip(url: str, folder: str):
logger.get_logger().info(f"extracted into {folder}")


def download_dns_yaml(url: str, folder: str):
def download_dns_yaml(url: str, folder: str, headers: Dict[str, str]):
logger.get_logger().info(f"downloading {url}")
response = requests.get(url, stream=True, timeout=(5, 10))
response = requests.get(url, headers=headers, stream=True, timeout=(100, 3))
with open(os.path.join(folder, "dns.yaml"), "wb") as out_file:
out_file.write(response.content)
logger.get_logger().info(f"downloaded into {folder}")


def download_all_assets(tmpdir: str, version="0.2.7"):
def download_all_assets(tmpdir: str, version="0.2.8"):
import subprocess
import platform

Expand Down Expand Up @@ -150,20 +158,13 @@ def download_all_assets(tmpdir: str, version="0.2.7"):
os.chmod(cmdfile, 0o755)
subprocess.run([cmdfile, "-notui", "-w", "0", "assets/chtts"])
except Exception:
BASE_URL = "https://raw.gitcode.com/u011570312/RVC-Models-Downloader/assets/"
suffix = {
"darwin_amd64": "987",
"darwin_arm64": "988",
"linux_386": "989",
"linux_amd64": "990",
"linux_arm64": "991",
"windows_386": "992",
"windows_amd64": "993",
}[f"{system_type}_{architecture}"]
RVCMD_URL = BASE_URL + suffix
BASE_URL = "https://gitea.seku.su/fumiama/RVC-Models-Downloader/releases/download/"
suffix = "zip" if is_win else "tar.gz"
RVCMD_URL = BASE_URL + f"v{version}/rvcmd_{system_type}_{architecture}.{suffix}"
download_dns_yaml(
"https://raw.gitcode.com/u011570312/RVC-Models-Downloader/raw/main/dns.yaml",
"https://gitea.seku.su/fumiama/RVC-Models-Downloader/raw/branch/main/dns.yaml",
tmpdir,
headers={"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 Edg/128.0.0.0"},
)
if is_win:
download_and_extract_zip(RVCMD_URL, tmpdir)
Expand Down
14 changes: 10 additions & 4 deletions tools/checksum/tmpl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,25 @@ package main
var files = [...]string{
"asset/Decoder.pt",
"asset/DVAE_full.pt",
"asset/GPT.pt",
"asset/Embed.safetensors",
"asset/Vocos.pt",

"asset/gpt/config.json",
"asset/gpt/model.safetensors",

"asset/tokenizer/special_tokens_map.json",
"asset/tokenizer/tokenizer_config.json",
"asset/tokenizer/tokenizer.json",
}

const jsontmpl = `{
"sha256_asset_Decoder_pt" : "%s",
"sha256_asset_Decoder_pt" : "%s",
"sha256_asset_DVAE_full_pt" : "%s",
"sha256_asset_GPT_pt" : "%s",
"sha256_asset_Vocos_pt" : "%s",
"sha256_asset_Embed_safetensors" : "%s",
"sha256_asset_Vocos_pt" : "%s",
"sha256_asset_gpt_config_json" : "%s",
"sha256_asset_gpt_model_safetensors" : "%s",
"sha256_asset_tokenizer_special_tokens_map_json": "%s",
"sha256_asset_tokenizer_tokenizer_config_json" : "%s",
Expand Down

0 comments on commit c06f1d4

Please sign in to comment.