diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63de2dc..555179a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,4 +25,4 @@ jobs: pdm sync -d - name: Run Tests run: | - CRYPTOCOMPARE_APIKEY=${{secrets.CRYPTOCOMPARE_APIKEY}} pdm run all + CRYPTOCOMPARE_APIKEY=${{secrets.CRYPTOCOMPARE_APIKEY}} MINTSCAN_APIKEY=${{secrets.MINTSCAN_APIKEY}} pdm run all diff --git a/README.md b/README.md index 2f84bae..f207fd9 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,96 @@ # DHK Tokens Airdrop Scripts -A [Python script](./notebooks/airdrop.ipynb) to retrieve historical token prices and compute DHK tokens to distribute out each month. +A Python script to retrieve historical token prices and compute DHK tokens to distribute out each month. -- [DHK airdrop template master speadsheet](https://docs.google.com/spreadsheets/d/1QliDXE6yMNnPxhqraLqhTDnRQ0vbpvapLYoMC0vFgSc/edit?usp=sharing) +Output example -## Data Sources +![ss-example](./docs/ss-example.png) -Token price: -- +The key output is the table at the bottom. It can also be exported as csv or json. -Staking APR: -- (operated by [Cosmostation](https://cosmostation.io/)) +It is fine to see error messages as above saying unable to fetch price and staking apr for some tokens, as they may not be popular enough or not indexed by the service providers we used. -## Development - -Please copy `.env.example` over to `.env` and fills in the API_KEYs inside. +## Usage ```bash -pdm sync -d # sync all the dependencies -pdm exe --help # show the help file -pdm exe input.json -o output -t json +pip install dhkdao-airdrop + +# Get help +dhkdao-airdrop --help + +# Run script with api keys set in config.json +dhkdao_airdrop config.json -o output -t type + +# Run script with api keys set in env vars +CRYPTOCOMPARE_APIKEY="cc-apikey" MINTSCAN_APIKEY="ms-apikey" dhkdao_airdrop config.json -o output -t type ``` -- accept a file path, the input config file in json format. -- flag `-o`: accept a file path, the output file path. -- flag `-t`: accept one of the ["table", "json", "csv"], specifying the output format. +Options: + +- `-o`: Output file path. If skipped, output is printed on screen. +- `-t`: [table|csv|json]. Output type. + +You can have api keys set in the input config file or env var. Env vars will override the one set in config file. + +### Input Config + +An input config example is follows. + +```json +{ + "dhk_distribution": 100000, + "reference_date": "2024-08-27", + "tokens": [ + { "token": "AKT", "network": "akash", "qty": 188787 }, + { "token": "ATOM", "network": "cosmos", "qty": 592015 } + ], + "apis": { + "cryptocompare": { + "endpoint": "https://min-api.cryptocompare.com/data/pricehistorical", + "apikey": "cryptocompare-123456" + }, + "mintscan": { + "endpoint": "https://apis.mintscan.io/v1/:network/apr", + "apikey": "mintscan-123456" + } + } +} +``` + +- The config used to run for **2024 Sep** output is [shown here](./configs/config-202409.json), with the two `apikey` values redacted. + +- Most of the time, you only need to change the `reference_date` and the content inside `tokens`, after you have set the two `apikey` values correctly. -## Command +- For each token inside `tokens`, you also need to provide a `network` value, which is different from the token name. This is required by [Cosmostation API](https://docs.cosmostation.io/apis/reference/utilities/staking-apr). If an incorrect network value is provided, the staking APR will not be fetched. You can test the network value in the API panel provided by the link above. Another way is going to [Mintscan](https://www.mintscan.io/) and search for the token/network, the white label underneath it, as circled below, is the network value used in Cosmostation. + + ![network value](./docs/ss-networkvalue.png) + +- If you couldn't get a network staking APR from Mintscan API but know it somewhere, you can set `{..., "staking-apr": 0.xxx}` manually. This will prevent the script from fetching it from Cosmostation. + + Same principle goes that if you couldn't get a token price from Cyptocompare API but know it somewhere, you can set `{..., "price": xx.xxx }` manually. This will override the token price. + + An example is [here](https://github.com/dhkdao/airdrop-scripts/blob/jc/dev/configs/config-202409.json#L10). + +## Data Sources + +- Token price: +- Staking APR: (operated by [Cosmostation](https://cosmostation.io/)) +- [DHK airdrop template master speadsheet](https://docs.google.com/spreadsheets/d/1QliDXE6yMNnPxhqraLqhTDnRQ0vbpvapLYoMC0vFgSc/edit?usp=sharing) + +## Development ```bash -pip install dhkdao-airdrop -CRYPTOCOMPARE_APIKEY=abc MINTSCAN_APIKEY=abc dhkdao-airdrop -c input.json -o output -t json +# sync all the dependencies +pdm sync -d + +# show the help text +pdm exe --help + +# Run the regular command +pdm exe input.json -o output -t json + +# Run test cases with no output capture +pdm test:no-capture ``` + +Please copy `.env.example` over to `.env` and fills in the API_KEYs inside before running `pdm test`. Otherwise, tests expected to pass will fail. diff --git a/configs/config-202409.json b/configs/config-202409.json new file mode 100644 index 0000000..5620f50 --- /dev/null +++ b/configs/config-202409.json @@ -0,0 +1,23 @@ +{ + "dhk_distribution": 100000, + "reference_date": "2024-09-21", + "tokens": [ + { "token": "AKT", "network": "akash", "qty": 188787 }, + { "token": "ATOM", "network": "cosmos", "qty": 592015 }, + { "token": "JUNO", "network": "juno-1", "qty": 84703 }, + { "token": "HASH", "network": "hash", "qty": 25020226 }, + { "token": "OSMO", "network": "osmosis-1", "qty": 862867 }, + { "token": "STARS", "price": 0.00817, "staking-apr": 0.1363, "qty": 779458 }, + { "token": "DSM", "network": "DSM", "qty": 4010125 } + ], + "apis": { + "cryptocompare": { + "endpoint": "https://min-api.cryptocompare.com/data/pricehistorical", + "apikey": "cryptocompare-123456" + }, + "mintscan": { + "endpoint": "https://apis.mintscan.io/v1/:network/apr", + "apikey": "mintscan-123456" + } + } +} diff --git a/docs/ss-example.png b/docs/ss-example.png new file mode 100644 index 0000000..834be34 Binary files /dev/null and b/docs/ss-example.png differ diff --git a/docs/ss-networkvalue.png b/docs/ss-networkvalue.png new file mode 100644 index 0000000..e40044d Binary files /dev/null and b/docs/ss-networkvalue.png differ diff --git a/notebooks/airdrop.ipynb b/notebooks/airdrop.ipynb deleted file mode 100644 index 7c53200..0000000 --- a/notebooks/airdrop.ipynb +++ /dev/null @@ -1,325 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "2d32dcc5-6920-4853-a003-5dc1f2155f8f", - "metadata": {}, - "source": [ - "## DHK Airdrop Script (2024 Sep)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "2d72cd3a-fe30-4b0d-a4c0-1c33d1bf773e", - "metadata": {}, - "outputs": [], - "source": [ - "# These are parameters used in the program below\n", - "from datetime import datetime, timezone\n", - "from dotenv import load_dotenv\n", - "from os import environ\n", - "\n", - "load_dotenv()\n", - "\n", - "DHK_DISTRIBUTION = 100000\n", - "# Date for the price reference\n", - "PRICE_REF_DATE = datetime(2024, 8, 27, 12, tzinfo=timezone.utc)\n", - "\n", - "TOKENS = [\n", - " { \"token\": \"AKT\", \"network\": \"akash\", \"qty\": 188787 },\n", - " { \"token\": \"ATOM\", \"network\": \"cosmos\", \"qty\": 592015 },\n", - " { \"token\": \"JUNO\", \"network\": \"juno-1\", \"qty\": 84703 },\n", - "\n", - " # manual price\n", - " # { \"token\": \"HASH\", \"price\": \"\", \"network\": \"hash\", \"qty\": 25020226 },\n", - " { \"token\": \"OSMO\", \"network\": \"osmosis-1\", \"qty\": 862867 },\n", - "\n", - " # manual price and staking-apr\n", - " { \"token\": \"STARS\", \"price\": (0.008472+0.007874)/2, \"staking-apr\": 0.1363, \"qty\": 779458 },\n", - " { \"token\": \"DSM\", \"price\": 0, \"staking-apr\": 0, \"qty\": 4010125 }, \n", - "]\n", - "\n", - "\n", - "APIS = {\n", - " \"cryptocompare\": {\n", - " \"endpoint\": \"https://min-api.cryptocompare.com/data/pricehistorical\",\n", - " \"apikey\": environ.get('CRYPTOCOMPARE_APIKEY'),\n", - " },\n", - " \"mintscan\": {\n", - " \"endpoint\": \"https://apis.mintscan.io/v1/:network/apr\",\n", - " \"apikey\": environ.get('MINTSCAN_APIKEY'),\n", - " },\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "4c242dbe-8086-46a1-9c4d-49da4e08abde", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
tokenpricestaking-amtstaking-valstaking-aprrewarddhk-distribution-pcdhk-distribution
0AKT2.860000188787.0539930.820.143877642.0514.48614485.0
1ATOM4.764000592015.02820359.460.1504424182.0679.14179140.0
2JUNO0.09800084703.08300.890.18551539.820.287287.0
3OSMO0.428400862867.0369652.220.085931753.135.9245924.0
4STARS0.008173779458.06370.510.1363868.30.162162.0
5DSM0.0000004010125.00.00.00000.00.0000.0
6TOTALNaNNaN3744613.9NaN535985.36100.000100000.0
\n", - "
" - ], - "text/plain": [ - " token price staking-amt staking-val staking-apr reward \\\n", - "0 AKT 2.860000 188787.0 539930.82 0.1438 77642.05 \n", - "1 ATOM 4.764000 592015.0 2820359.46 0.1504 424182.06 \n", - "2 JUNO 0.098000 84703.0 8300.89 0.1855 1539.82 \n", - "3 OSMO 0.428400 862867.0 369652.22 0.0859 31753.13 \n", - "4 STARS 0.008173 779458.0 6370.51 0.1363 868.3 \n", - "5 DSM 0.000000 4010125.0 0.0 0.0000 0.0 \n", - "6 TOTAL NaN NaN 3744613.9 NaN 535985.36 \n", - "\n", - " dhk-distribution-pc dhk-distribution \n", - "0 14.486 14485.0 \n", - "1 79.141 79140.0 \n", - "2 0.287 287.0 \n", - "3 5.924 5924.0 \n", - "4 0.162 162.0 \n", - "5 0.000 0.0 \n", - "6 100.000 100000.0 " - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Program to retrieve necessary token information for export\n", - "from requests import Request, Session\n", - "from requests.exceptions import ConnectionError, Timeout, TooManyRedirects\n", - "import json\n", - "import math\n", - "import pandas as pd\n", - "\n", - "# API doc: https://min-api.cryptocompare.com/documentation?key=Historical&cat=dataPriceHistorical\n", - "def fetch_price(from_symbol, to_symbol='USD', date=PRICE_REF_DATE):\n", - " endpoint, apikey = APIS[\"cryptocompare\"][\"endpoint\"], APIS[\"cryptocompare\"][\"apikey\"]\n", - " parameters = {\n", - " 'fsym':from_symbol,\n", - " 'tsyms': to_symbol,\n", - " 'calculationType':'MidHighLow',\n", - " 'ts': date.timestamp()\n", - " }\n", - " headers = {\n", - " 'Accepts': 'application/json',\n", - " 'authorization': f\"Apikey {apikey}\",\n", - " }\n", - "\n", - " session = Session()\n", - " session.headers.update(headers)\n", - " try:\n", - " response = session.get(endpoint, params=parameters)\n", - " data = json.loads(response.text)\n", - " except (ConnectionError, Timeout, TooManyRedirects) as e:\n", - " raise Exception(f\"fetch_price connection error: {e}\")\n", - "\n", - " if(from_symbol not in data):\n", - " raise Exception(f\"{from_symbol}: unable to fetch price for, returning: {data}\")\n", - " \n", - " return data[from_symbol][to_symbol] \n", - "\n", - "\n", - "# API doc: https://docs.cosmostation.io/apis/reference/utilities/staking-apr\n", - "def fetch_staking_apr(network):\n", - " endpoint, apikey = APIS[\"mintscan\"][\"endpoint\"], APIS[\"mintscan\"][\"apikey\"]\n", - "\n", - " endpoint = endpoint.replace(\":network\", network)\n", - " headers = {\n", - " 'Accepts': 'application/json',\n", - " 'authorization': f\"Bearer {apikey}\",\n", - " }\n", - "\n", - " session = Session()\n", - " session.headers.update(headers)\n", - "\n", - " try:\n", - " response = session.get(endpoint)\n", - " data = json.loads(response.text)\n", - " except (ConnectionError, Timeout, TooManyRedirects) as e:\n", - " raise Exception(f\"fetch_staking_apr connection error: {e}\")\n", - "\n", - " if(\"apr\" not in data):\n", - " raise Exception(f\"{network}: unable to fetch staking_apr, returning: {data}\")\n", - " \n", - " return float(data[\"apr\"])\n", - " \n", - "\n", - "def get_main_table():\n", - " columns = [\n", - " \"token\", \"price\", \"staking-amt\", \"staking-val\", \"staking-apr\", \n", - " \"reward\", \"dhk-distribution-pc\", \"dhk-distribution\"\n", - " ]\n", - " lst = []\n", - " \n", - " for t in TOKENS:\n", - " token, staking_amt = t[\"token\"], t[\"qty\"]\n", - " token_price = t[\"price\"] if \"price\" in t else round(fetch_price(token), 5)\n", - " \n", - " staking_val = round(token_price * staking_amt, 2)\n", - " staking_apr = 0\n", - " if \"staking-apr\" in t:\n", - " staking_apr = t[\"staking-apr\"]\n", - " elif \"network\" in t:\n", - " staking_apr = round(fetch_staking_apr(t[\"network\"]), 4)\n", - " \n", - " reward = round(staking_apr * staking_val, 2)\n", - "\n", - " lst.append([token, token_price, staking_amt, staking_val, staking_apr, reward, 0, 0])\n", - "\n", - " mt = pd.DataFrame(lst, columns=columns)\n", - " \n", - " # Append a total row at the end of the table\n", - " ttl_staking_val = mt[\"staking-val\"].sum()\n", - " ttl_reward = mt[\"reward\"].sum()\n", - " ttl = pd.Series({\"token\": \"TOTAL\", \"staking-val\": ttl_staking_val, \"reward\": ttl_reward})\n", - " mt = pd.concat([mt, ttl.to_frame().T], ignore_index=True)\n", - "\n", - " # Calculate DHK distribution and distribution percent\n", - " for idx, row in mt.iterrows():\n", - " mt.at[idx, \"dhk-distribution-pc\"] = round((row[\"reward\"] / ttl_reward) * 100, 3)\n", - " mt.at[idx, \"dhk-distribution\"] = math.floor(row[\"reward\"] / ttl_reward * DHK_DISTRIBUTION)\n", - " \n", - " return mt\n", - "\n", - "get_main_table()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pyproject.toml b/pyproject.toml index be16271..6a62180 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["pdm-backend"] build-backend = "pdm.backend" [project] -name = "dhkdao_airdrop_scripts" +name = "dhkdao-airdrop" version = "0.1.0" authors = [ { name = "Jimmy Chu", email = "jimmychu0807@gmail.com" }, @@ -39,9 +39,9 @@ Issues = "https://github.com/dhkdao/airdrop-scripts/issues" distribution = true [tool.pdm.scripts] -test = "pytest --capture=no tests" -"test:ci" = "pytest" +test = "pytest" +"test:no-capture" = "pytest --capture=no tests" lint = "flake8 src tests" "lint:write" = "black src tests" exe = "pdm run src/scripts/main.py" -all = { composite = ["lint", "test:ci"]} +all = { composite = ["lint", "test"]} diff --git a/src/scripts/__init__.py b/src/scripts/__init__.py index 5e759a9..715e7ff 100644 --- a/src/scripts/__init__.py +++ b/src/scripts/__init__.py @@ -1,4 +1,4 @@ from .airdrop import Airdrop -from .utils import is_number, round_output +from .utils import is_number, round_output, get_config_with_apikey_envs -__all__ = ["Airdrop", "is_number", "round_output"] +__all__ = ["Airdrop", "is_number", "round_output", "get_config_with_apikey_envs"] diff --git a/src/scripts/airdrop.py b/src/scripts/airdrop.py index ab8a3a3..252f7f7 100644 --- a/src/scripts/airdrop.py +++ b/src/scripts/airdrop.py @@ -174,11 +174,13 @@ def monthly_alloc(self): # Calculate DHK distribution and distribution percent for idx, row in df.iterrows(): df.at[idx, "dhk-distribution-pc"] = ( - (row["reward"] / ttl_reward) * 100 if is_number(row["reward"]) else None + (row["reward"] / ttl_reward) * 100 + if is_number(row["reward"]) and ttl_reward > 0 + else None ) df.at[idx, "dhk-distribution"] = ( (row["reward"] / ttl_reward) * self.config["dhk_distribution"] - if is_number(row["reward"]) + if is_number(row["reward"]) and ttl_reward > 0 else None ) diff --git a/src/scripts/main.py b/src/scripts/main.py index 9e59bfb..8db961f 100644 --- a/src/scripts/main.py +++ b/src/scripts/main.py @@ -1,10 +1,9 @@ -import json import typer from typing_extensions import Annotated from enum import Enum from rich import print -from scripts import round_output, Airdrop +from scripts import Airdrop, round_output, get_config_with_apikey_envs class OutputType(str, Enum): @@ -32,9 +31,10 @@ def monthly_alloc( ] = OutputType.table, ): """ - Compute the DHK dao monthly airdrop allocation based on staked value on various blockchains. + Compute the DHKdao monthly airdrop allocation based on staked value on various blockchains. """ - airdrop = Airdrop(json.loads(config.read())) + config = get_config_with_apikey_envs(config) + airdrop = Airdrop(config) result = round_output(airdrop.monthly_alloc()) result_output = None diff --git a/src/scripts/utils.py b/src/scripts/utils.py index fc6aab3..c4b413a 100644 --- a/src/scripts/utils.py +++ b/src/scripts/utils.py @@ -1,3 +1,8 @@ +import io +import json +import os + + def round_number(val): if isinstance(val, float): if val < 1: # keep 4 decimals @@ -17,3 +22,16 @@ def round_output(df): def is_number(i): return isinstance(i, (int, float)) + + +def get_config_with_apikey_envs(config): + if isinstance(config, io.TextIOWrapper): + config = json.loads(config.read()) + + if os.getenv("CRYPTOCOMPARE_APIKEY") is not None: + config["apis"]["cryptocompare"]["apikey"] = os.getenv("CRYPTOCOMPARE_APIKEY") + + if os.getenv("MINTSCAN_APIKEY") is not None: + config["apis"]["mintscan"]["apikey"] = os.getenv("MINTSCAN_APIKEY") + + return config diff --git a/tests/scripts/test_airdrop.py b/tests/scripts/test_airdrop.py index 79a93b2..dcce238 100644 --- a/tests/scripts/test_airdrop.py +++ b/tests/scripts/test_airdrop.py @@ -3,16 +3,19 @@ import pytest import typer from dotenv import load_dotenv -from scripts import Airdrop +from scripts import Airdrop, get_config_with_apikey_envs load_dotenv() -UNKNOWN_TOKEN = "HASH" -KNOWN_TOKEN = "ATOM" INVALID_CONFIG_FILEPATH = os.path.join( os.path.dirname(__file__), "../fixtures/config-invalid-1.json" ) +KNOWN_TOKEN = "ATOM" +UNKNOWN_TOKEN = "HASH" +KNOWN_NETWORK = "akash" +UNKNOWN_NETWORK = "hash" + print("INVALID_CONFIG_FILEPATH", INVALID_CONFIG_FILEPATH) @@ -38,25 +41,7 @@ def get_config(): ) -def get_config_with_cryptocompare_apikey_env(): - config = get_config() - config["apis"]["cryptocompare"]["apikey"] = os.getenv("CRYPTOCOMPARE_APIKEY") - return config - - class TestAirdropClass: - def test_fetch_price_on_unknown_token_should_return_none(self): - config = get_config_with_cryptocompare_apikey_env() - airdrop = Airdrop(config) - price = airdrop.fetch_price(UNKNOWN_TOKEN) - assert price is None - - def test_fetch_price_on_known_token_should_return_value(self): - config = get_config_with_cryptocompare_apikey_env() - airdrop = Airdrop(config) - price = airdrop.fetch_price(KNOWN_TOKEN) - assert isinstance(price, float) - def test_on_config_missing_key_should_raise_exception(self): with open(INVALID_CONFIG_FILEPATH, "r") as f: config = json.loads(f.read()) @@ -66,5 +51,26 @@ def test_on_config_missing_key_should_raise_exception(self): assert excinfo.type is typer.Exit - def test_fetch_staking_apr(self): - pass + def test_fetch_price_on_known_token_should_work(self): + config = get_config_with_apikey_envs(get_config()) + airdrop = Airdrop(config) + price = airdrop.fetch_price(KNOWN_TOKEN) + assert isinstance(price, float) + + def test_fetch_price_on_unknown_token_should_return_none(self): + config = get_config_with_apikey_envs(get_config()) + airdrop = Airdrop(config) + price = airdrop.fetch_price(UNKNOWN_TOKEN) + assert price is None + + def test_fetch_staking_apr_on_known_network_should_work(self): + config = get_config_with_apikey_envs(get_config()) + airdrop = Airdrop(config) + staking_apr = airdrop.fetch_staking_apr(KNOWN_NETWORK) + assert isinstance(staking_apr, float) + + def test_fetch_staking_apr_on_unknown_network_should_return_none(self): + config = get_config_with_apikey_envs(get_config()) + airdrop = Airdrop(config) + staking_apr = airdrop.fetch_staking_apr(UNKNOWN_NETWORK) + assert staking_apr is None