diff --git a/backend/moonstream/actions.py b/backend/moonstream/actions.py index 64bcdbfa..22612cd8 100644 --- a/backend/moonstream/actions.py +++ b/backend/moonstream/actions.py @@ -1,13 +1,14 @@ import json import logging from typing import Dict, Any, List, Optional - +from enum import Enum import boto3 # type: ignore from moonstreamdb.models import ( EthereumAddress, EthereumLabel, ) -from sqlalchemy.orm import Session +from sqlalchemy import text +from sqlalchemy.orm import Session, query, query_expression from . import data from .settings import ETHERSCAN_SMARTCONTRACTS_BUCKET @@ -50,6 +51,62 @@ def get_contract_source_info( return None +class LabelNames(Enum): + ETHERSCAN_SMARTCONTRACT = "etherscan_smartcontract" + COINMARKETCAP_TOKEN = "coinmarketcap_token" + + +def get_ethereum_address_info( + db_session: Session, address: str +) -> Optional[data.EthereumAddressInfo]: + query = db_session.query(EthereumAddress.id).filter( + EthereumAddress.address == address + ) + id = query.one_or_none() + if id is None: + return None + + address_info = data.EthereumAddressInfo(address=address) + etherscan_address_url = f"https://etherscan.io/address/{address}" + blockchain_com_url = f"https://www.blockchain.com/eth/address/{address}" + # Checking for token: + coinmarketcap_label: Optional[EthereumLabel] = ( + db_session.query(EthereumLabel) + .filter(EthereumLabel.address_id == id[0]) + .filter(EthereumLabel.label == LabelNames.COINMARKETCAP_TOKEN.value) + .order_by(text("created_at desc")) + .limit(1) + .one_or_none() + ) + if coinmarketcap_label is not None: + address_info.token = data.EthereumTokenDetails( + name=coinmarketcap_label.label_data["name"], + symbol=coinmarketcap_label.label_data["symbol"], + external_url=[ + coinmarketcap_label.label_data["coinmarketcap_url"], + etherscan_address_url, + blockchain_com_url, + ], + ) + + # Checking for smart contract + etherscan_label: Optional[EthereumLabel] = ( + db_session.query(EthereumLabel) + .filter(EthereumLabel.address_id == id[0]) + .filter(EthereumLabel.label == LabelNames.ETHERSCAN_SMARTCONTRACT.value) + .order_by(text("created_at desc")) + .limit(1) + .one_or_none() + ) + if etherscan_label is not None: + address_info.smart_contract = data.EthereumSmartContractDetails( + name=etherscan_label.label_data["name"], + external_url=[etherscan_address_url, blockchain_com_url], + ) + + return address_info + + def get_address_labels( db_session: Session, start: int, limit: int, addresses: Optional[str] = None ) -> data.AddressListLabelsResponse: diff --git a/backend/moonstream/api.py b/backend/moonstream/api.py index 6ae117ff..48ab8cda 100644 --- a/backend/moonstream/api.py +++ b/backend/moonstream/api.py @@ -12,6 +12,7 @@ from .routes.subscriptions import app as subscriptions_api from .routes.users import app as users_api from .routes.txinfo import app as txinfo_api from .routes.streams import app as streams_api +from .routes.address_info import app as addressinfo_api from .settings import ORIGINS from .version import MOONSTREAM_VERSION @@ -49,3 +50,4 @@ app.mount("/subscriptions", subscriptions_api) app.mount("/users", users_api) app.mount("/streams", streams_api) app.mount("/txinfo", txinfo_api) +app.mount("/address_info", addressinfo_api) diff --git a/backend/moonstream/data.py b/backend/moonstream/data.py index b58b871a..e8e36c5e 100644 --- a/backend/moonstream/data.py +++ b/backend/moonstream/data.py @@ -157,6 +157,23 @@ class EthereumSmartContractSourceInfo(BaseModel): compiler_version: str +class EthereumTokenDetails(BaseModel): + name: Optional[str] + symbol: Optional[str] + external_url: List[str] = [] + + +class EthereumSmartContractDetails(BaseModel): + name: Optional[str] + external_url: List[str] = [] + + +class EthereumAddressInfo(BaseModel): + address: str + token: Optional[EthereumTokenDetails] + smart_contract: Optional[EthereumSmartContractDetails] + + class TxinfoEthereumBlockchainResponse(BaseModel): tx: EthereumTransaction is_smart_contract_deployment: bool = False diff --git a/backend/moonstream/providers/bugout.py b/backend/moonstream/providers/bugout.py index 57b7d1fb..4ba9629e 100644 --- a/backend/moonstream/providers/bugout.py +++ b/backend/moonstream/providers/bugout.py @@ -91,6 +91,7 @@ class BugoutEventProvider: """ is_query_constrained = query.subscription_types or query.subscriptions relevant_subscriptions = user_subscriptions.get(self.event_type) + if ( is_query_constrained and self.event_type not in query.subscription_types ) or not relevant_subscriptions: @@ -268,10 +269,46 @@ class BugoutEventProvider: return self.entry_event(search_results.results[0]) +class EthereumTXPoolProvider(BugoutEventProvider): + def __init__( + self, + event_type: str, + tags: Optional[List[str]] = None, + batch_size: int = 100, + timeout: float = 30.0, + ): + + super().__init__(event_type, tags, batch_size, timeout) + + def parse_filters( + self, query: StreamQuery, user_subscriptions: Dict[str, List[BugoutResource]] + ) -> Optional[List[str]]: + + is_query_constrained = query.subscription_types or query.subscriptions + relevant_subscriptions = user_subscriptions.get(self.event_type) + + if ( + is_query_constrained and self.event_type not in query.subscription_types + ) or not relevant_subscriptions: + return None + addresses = [ + subscription.resource_data["address"] + for subscription in relevant_subscriptions + ] + subscriptions_filters = [] + for adress in addresses: + subscriptions_filters.extend( + [f"?#from_address:{adress}", f"?#to_address:{adress}"] + ) + + return subscriptions_filters + + whalewatch_provider = BugoutEventProvider( event_type="ethereum_whalewatch", tags=["crawl_type:ethereum_trending"] ) -ethereum_txpool_provider = BugoutEventProvider( + +ethereum_txpool_provider = EthereumTXPoolProvider( event_type="ethereum_txpool", tags=["client:ethereum-txpool-crawler-0"] ) diff --git a/backend/moonstream/routes/address_info.py b/backend/moonstream/routes/address_info.py new file mode 100644 index 00000000..2b95bb92 --- /dev/null +++ b/backend/moonstream/routes/address_info.py @@ -0,0 +1,87 @@ +import logging +from typing import Dict, List, Optional + +from sqlalchemy.sql.expression import true + +from fastapi import FastAPI, Depends, Query, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from moonstreamdb.db import yield_db_session +from sqlalchemy.orm import Session + +from .. import actions +from .. import data +from ..middleware import BroodAuthMiddleware +from ..settings import DOCS_TARGET_PATH, ORIGINS, DOCS_PATHS +from ..version import MOONSTREAM_VERSION + +logger = logging.getLogger(__name__) + +tags_metadata = [ + {"name": "addressinfo", "description": "Address public information."}, + {"name": "labels", "description": "Addresses label information."}, +] + +app = FastAPI( + title=f"Moonstream users API.", + description="User, token and password handlers.", + version=MOONSTREAM_VERSION, + openapi_tags=tags_metadata, + openapi_url="/openapi.json", + docs_url=None, + redoc_url=f"/{DOCS_TARGET_PATH}", +) + +app.add_middleware( + CORSMiddleware, + allow_origins=ORIGINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +whitelist_paths: Dict[str, str] = {} +whitelist_paths.update(DOCS_PATHS) +app.add_middleware(BroodAuthMiddleware, whitelist=whitelist_paths) + + +@app.get( + "/ethereum_blockchain", + tags=["addressinfo"], + response_model=data.EthereumAddressInfo, +) +async def addressinfo_handler( + address: str, + db_session: Session = Depends(yield_db_session), +) -> Optional[data.EthereumAddressInfo]: + response = actions.get_ethereum_address_info(db_session, address) + return response + + +@app.get( + "/labels/ethereum_blockchain", + tags=["labels bul"], + response_model=data.AddressListLabelsResponse, +) +async def addresses_labels_bulk_handler( + addresses: Optional[str] = Query(None), + start: int = Query(0), + limit: int = Query(100), + db_session: Session = Depends(yield_db_session), +) -> data.AddressListLabelsResponse: + """ + Fetch labels with additional public information + about known addresses. + """ + if limit > 100: + raise HTTPException( + status_code=406, detail="The limit cannot exceed 100 addresses" + ) + try: + addresses_response = actions.get_address_labels( + db_session=db_session, start=start, limit=limit, addresses=addresses + ) + except Exception as err: + logger.error(f"Unable to get info about Ethereum addresses {err}") + raise HTTPException(status_code=500) + + return addresses_response diff --git a/backend/moonstream/routes/txinfo.py b/backend/moonstream/routes/txinfo.py index 92562049..20eec3c7 100644 --- a/backend/moonstream/routes/txinfo.py +++ b/backend/moonstream/routes/txinfo.py @@ -10,7 +10,7 @@ from typing import Dict, Optional from sqlalchemy.sql.expression import true -from fastapi import FastAPI, Depends, HTTPException, Query +from fastapi import FastAPI, Depends from fastapi.middleware.cors import CORSMiddleware from moonstreamdb.db import yield_db_session from moonstreamdb.models import EthereumAddress @@ -27,7 +27,6 @@ logger = logging.getLogger(__name__) tags_metadata = [ {"name": "txinfo", "description": "Ethereum transactions info."}, - {"name": "address info", "description": "Addresses public information."}, ] app = FastAPI( @@ -92,31 +91,3 @@ async def txinfo_ethereum_blockchain_handler( response.smart_contract_address = txinfo_request.tx.to_address response.is_smart_contract_call = True return response - - -@app.get( - "/addresses", tags=["address info"], response_model=data.AddressListLabelsResponse -) -async def addresses_labels_handler( - addresses: Optional[str] = Query(None), - start: int = Query(0), - limit: int = Query(100), - db_session: Session = Depends(yield_db_session), -) -> data.AddressListLabelsResponse: - """ - Fetch labels with additional public information - about known addresses. - """ - if limit > 100: - raise HTTPException( - status_code=406, detail="The limit cannot exceed 100 addresses" - ) - try: - addresses_response = actions.get_address_labels( - db_session=db_session, start=start, limit=limit, addresses=addresses - ) - except Exception as err: - logger.error(f"Unable to get info about Ethereum addresses {err}") - raise HTTPException(status_code=500) - - return addresses_response diff --git a/crawlers/ethtxpool/main.go b/crawlers/ethtxpool/main.go index 2c0b7ab5..224ae7dc 100644 --- a/crawlers/ethtxpool/main.go +++ b/crawlers/ethtxpool/main.go @@ -138,6 +138,7 @@ func PollTxpoolContent(gethClient *rpc.Client, interval int, reporter *humbug.Hu fmt.Sprintf("max_priority_fee_per_gas:%d", pendingTx.Transaction.MaxPriorityFeePerGas.ToInt()), fmt.Sprintf("max_fee_per_gas:%d", pendingTx.Transaction.MaxFeePerGas.ToInt()), fmt.Sprintf("gas:%d", pendingTx.Transaction.Gas), + "crawl_type:ethereum_txpool", } report := humbug.Report{ Title: ReportTitle, diff --git a/crawlers/mooncrawl/.mooncrawl/bin/python b/crawlers/mooncrawl/.mooncrawl/bin/python deleted file mode 120000 index b8a0adbb..00000000 --- a/crawlers/mooncrawl/.mooncrawl/bin/python +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/crawlers/mooncrawl/.mooncrawl/bin/python3 b/crawlers/mooncrawl/.mooncrawl/bin/python3 deleted file mode 120000 index ae65fdaa..00000000 --- a/crawlers/mooncrawl/.mooncrawl/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -/usr/bin/python3 \ No newline at end of file diff --git a/crawlers/mooncrawl/.mooncrawl/bin/python3.9 b/crawlers/mooncrawl/.mooncrawl/bin/python3.9 deleted file mode 120000 index b8a0adbb..00000000 --- a/crawlers/mooncrawl/.mooncrawl/bin/python3.9 +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/crawlers/mooncrawl/.mooncrawl/lib64 b/crawlers/mooncrawl/.mooncrawl/lib64 deleted file mode 120000 index 7951405f..00000000 --- a/crawlers/mooncrawl/.mooncrawl/lib64 +++ /dev/null @@ -1 +0,0 @@ -lib \ No newline at end of file diff --git a/crawlers/mooncrawl/.mooncrawl/pyvenv.cfg b/crawlers/mooncrawl/.mooncrawl/pyvenv.cfg deleted file mode 100644 index 661bca34..00000000 --- a/crawlers/mooncrawl/.mooncrawl/pyvenv.cfg +++ /dev/null @@ -1,3 +0,0 @@ -home = /home/neeraj/external/python/Python-3.9.6/bin -include-system-site-packages = false -version = 3.9.6 diff --git a/crawlers/mooncrawl/mooncrawl/identity.py b/crawlers/mooncrawl/mooncrawl/identity.py index ff11d267..c91b8d49 100644 --- a/crawlers/mooncrawl/mooncrawl/identity.py +++ b/crawlers/mooncrawl/mooncrawl/identity.py @@ -1,4 +1,5 @@ import argparse +import logging import os import time @@ -8,6 +9,9 @@ from sqlalchemy import text from moonstreamdb.db import yield_db_session_ctx from moonstreamdb.models import EthereumAddress, EthereumLabel +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + COINMARKETCAP_API_KEY = os.environ.get("COINMARKETCAP_API_KEY") if COINMARKETCAP_API_KEY is None: raise ValueError("COINMARKETCAP_API_KEY environment variable must be set") @@ -50,45 +54,70 @@ def identities_cmc_add_handler(args: argparse.Namespace) -> None: raise Exception(err) if len(response["data"]) == 0: - print("No more data, crawling finished") + logger.info("No more data, crawling finished") break with yield_db_session_ctx() as db_session: - latest_address = ( + latest_address = 1 + latest_address_obj = ( db_session.query(EthereumAddress.id) .order_by(text("id desc")) .limit(1) - .one() - )[0] + .one_or_none() + ) + if latest_address_obj is not None: + latest_address = latest_address_obj[0] + for coin in response["data"]: if coin["platform"] is not None: if ( coin["platform"]["id"] == 1027 and coin["platform"]["token_address"] is not None ): - latest_address += 1 - eth_token_id = latest_address - eth_token = EthereumAddress( - id=eth_token_id, - address=coin["platform"]["token_address"], + token_address = coin["platform"]["token_address"] + # Check if address already exists + address = ( + db_session.query(EthereumAddress) + .filter(EthereumAddress.address == token_address) + .one_or_none() ) - db_session.add(eth_token) - eth_token_label = EthereumLabel( - label="coinmarketcap_token", - address_id=eth_token_id, - label_data={ - "name": coin["name"], - "symbol": coin["symbol"], - "coinmarketcap_url": f'https://coinmarketcap.com/currencies/{coin["slug"]}', - }, + # Add new address + if address is None: + latest_address += 1 + eth_token_id = latest_address + eth_token = EthereumAddress( + id=eth_token_id, + address=token_address, + ) + db_session.add(eth_token) + logger.info(f"Added {coin['name']} token") + else: + eth_token_id = address.id + + label = ( + db_session.query(EthereumLabel) + .filter(EthereumLabel.address_id == eth_token_id) + .one_or_none() ) - db_session.add(eth_token_label) - print(f"Added {coin['name']} token") + if label is None: + eth_token_label = EthereumLabel( + label="coinmarketcap_token", + address_id=eth_token_id, + label_data={ + "name": coin["name"], + "symbol": coin["symbol"], + "coinmarketcap_url": f'https://coinmarketcap.com/currencies/{coin["slug"]}', + }, + ) + db_session.add(eth_token_label) + logger.info(f"Added label for {coin['name']} token") db_session.commit() start_n += limit_n - print(f"Loop ended, starting new from {start_n} to {start_n + limit_n - 1}") + logger.info( + f"Loop ended, starting new from {start_n} to {start_n + limit_n - 1}" + ) time.sleep(1) diff --git a/crawlers/mooncrawl/sample.env b/crawlers/mooncrawl/sample.env index 4530ebab..5ebad6e0 100644 --- a/crawlers/mooncrawl/sample.env +++ b/crawlers/mooncrawl/sample.env @@ -2,8 +2,7 @@ export MOONSTREAM_IPC_PATH=null export MOONSTREAM_CRAWL_WORKERS=4 export MOONSTREAM_DB_URI="postgresql://:@:/" - -export MOONSTREAM_ETHERSCAN_TOKEN="TOKEN" -export AWS_S3_SMARTCONTRACT_BUCKET="" +export MOONSTREAM_ETHERSCAN_TOKEN="" +export AWS_S3_SMARTCONTRACT_BUCKET="" export MOONSTREAM_HUMBUG_TOKEN="" export COINMARKETCAP_API_KEY="" diff --git a/frontend/pages/_app.js b/frontend/pages/_app.js index 44ad346c..b4e2894c 100644 --- a/frontend/pages/_app.js +++ b/frontend/pages/_app.js @@ -21,6 +21,7 @@ const DefaultLayout = dynamic(() => import("../src/layouts"), { }); import { useRouter } from "next/router"; import NProgress from "nprogress"; +import { WHITE_LOGO_W_TEXT_URL } from "../src/core/constants"; export default function CachingApp({ Component, pageProps }) { const [queryClient] = useState(new QueryClient()); @@ -48,6 +49,10 @@ export default function CachingApp({ Component, pageProps }) { const getLayout = Component.getLayout || ((page) => {page}); + const headLinks = [ + { rel: "preload", as: "image", href: WHITE_LOGO_W_TEXT_URL }, + ]; + pageProps.preloads && headLinks.push(...pageProps.preloads); return ( <> {pageProps.metaTags && } - {pageProps.preloads && } + {getLayout()} diff --git a/frontend/pages/product/index.js b/frontend/pages/product/index.js index 1057ec3d..55c6376a 100644 --- a/frontend/pages/product/index.js +++ b/frontend/pages/product/index.js @@ -7,8 +7,6 @@ import { Stack, chakra, useMediaQuery, - UnorderedList, - ListItem, useBreakpointValue, } from "@chakra-ui/react"; import { DEFAULT_METATAGS, AWS_ASSETS_PATH } from "../../src/core/constants"; @@ -133,47 +131,54 @@ const Product = () => { alignItems="center" pb={24} > - - - {`Why you'll love moonstream`} + + + {`Why you'll love Moonstream`} - + We strive for financial inclusion. With cryptocurrencies becoming mainstream, now is the time for anyone with a computer and access to the Internet to utilize this opportunity to make passive income. We’re here to make it easier. - + Right now our source of data is Ethereum blockchain. Our goal is to provide a live view of the transactions taking place on every public blockchain - from the activity of specific accounts or smart contracts to updates about general market movements. - + This information comes from the blockchains themselves, from their mempools/transaction pools, and from centralized exchanges, social media, and the news. This forms a stream of information tailored to your specific needs. - + We’re giving you a macro view of the crypto market with direct access from Moonstream dashboards to execute transactions. You can also set up programs which execute (on- or off-chain) when your stream meets certain conditions. - + Moonstream is accessible through dashboard, API and webhooks. - + Moonstream’s financial inclusion goes beyond providing access to data. All of our work is open source as we do not believe that proprietary technologies are financially inclusive. - + You can read{" "} @@ -181,7 +186,7 @@ const Product = () => { {" "} and keep track of our progress using{" "} @@ -191,82 +196,6 @@ const Product = () => { - - - Product - - - - Moonstream is a product which helps anyone participate in - decentralized finance. From the most sophisticated flash - arbitrageurs to people looking for yield from currency that would - otherwise lie dormant in their exchange accounts. - - - We aim to go far beyond raw transaction information, enriching our - view with context from centralized exchanges, the news, social - media, and smart contract analysis. - - - Moonstream users can subscribe to events from any blockchain - from - the activity of specific accounts or smart contracts to updates - about general market movements. This information comes from the - blockchains themselves, from their mempools/transaction{" "} - pools, and from centralized exchanges, social media, and the news. - This forms a stream of information tailored to their specific needs. - - - They can use this information to execute transactions directly from - the Moonstream frontend or they can set up programs which execute - (on- or off-chain) when their stream meets certain conditions. - - - Moonstream will be accessible to software through our API and - webhooks. - - - Moonstream customers are: - - - Development teams deploying decentralized applications - - They use Moonstream to analyze how users are calling their - dapps, and set up alerts for suspicious activity.{" "} - - - Algorithmic funds - They use Moonstream to execute - transactions directly on-chain under prespecified conditions. - - - Crypto traders - They use Moonstream to evaluate trading - strategies based on data from centralized exchanges, the - blockchain, and the transaction pool. - - - - - Moonstream’s financial inclusion goes beyond providing access to - data. We also help validators and stakers on proof of stake chains - earn rewards in excess of the validation rewards. We pay validators - to send mempool/transaction pool data back to Moonstream, and they - divide these payments between themselves and their stakers. This - helps validators attract more stake on proof of stake blockchains - like Algorand, Solana, and post-merge Ethereum. It also ensures that - Moonstream users have access to the freshest and most geographically - diverse transaction pool data on the market. - - - All of our work is open source as we do not believe that proprietary - technologies are financially inclusive.{" "} - - You can read our code on GitHub. - - - - ); }; diff --git a/frontend/pages/team/index.js b/frontend/pages/team/index.js index 2500d844..e53eb209 100644 --- a/frontend/pages/team/index.js +++ b/frontend/pages/team/index.js @@ -155,7 +155,6 @@ const Product = () => { { const { toggleModal } = useModals(); return ( <> - <> - {ui.isMobileView && ( - <> - ui.setSidebarToggled(!ui.sidebarToggled)} - icon={} - /> - - )} - - - - Moonstream logo - - - + {ui.isMobileView && ( + <> + ui.setSidebarToggled(!ui.sidebarToggled)} + icon={} + /> + + )} + - {!ui.isMobileView && ( - <> - - - {ALL_NAV_PATHES.map((item, idx) => ( - - {item.title} - - ))} + {!ui.isMobileView && ( + <> + + + {ALL_NAV_PATHES.map((item, idx) => ( + + {item.title} + + ))} - {ui.isLoggedIn && ( - - - - )} - {!ui.isLoggedIn && ( + {ui.isLoggedIn && ( + - )} - {!ui.isLoggedIn && ( - - )} - {ui.isLoggedIn && ( - - )} - - - )} - {ui.isLoggedIn && ui.isMobileView && ( - <> - - - - )} - + + )} + {!ui.isLoggedIn && ( + + )} + {!ui.isLoggedIn && ( + + )} + {ui.isLoggedIn && ( + + )} + + + )} + {ui.isLoggedIn && ui.isMobileView && ( + <> + + + + )} ); }; diff --git a/frontend/src/components/Navbar.js b/frontend/src/components/Navbar.js index 20d9bedc..7d6cc257 100644 --- a/frontend/src/components/Navbar.js +++ b/frontend/src/components/Navbar.js @@ -19,16 +19,11 @@ const Navbar = () => { boxShadow={["sm", "md"]} alignItems="center" id="Navbar" - minH={["3rem", "3rem", "3rem", "3rem", "3rem", "3rem"]} - // overflow="initial" + minH="3rem" + maxH="3rem" bgColor="primary.1200" - // flexWrap="wrap" - direction={["row", "row", "row", null, "row"]} - // zIndex={100} + direction="row" w="100%" - minW="100%" - m={0} - p={0} overflow="hidden" >