import json, os, random, string
from datetime import datetime, timezone
websites = [
("NASA APIs","https://api.nasa.gov/","space science APIs"),
("NASA","https://www.nasa.gov/","space agency"),
("NASA Earthdata","https://www.earthdata.nasa.gov/","Earth observation data"),
("NASA JPL","https://www.jpl.nasa.gov/","planetary science"),
("NASA Image Library","https://images.nasa.gov/","space imagery"),
("Google","https://www.google.com/","search reference"),
("Google Developers","https://developers.google.com/","developer documentation"),
("YouTube","https://www.youtube.com/","video reference"),
("YouTube Data API","https://developers.google.com/youtube/v3","YouTube API docs"),
("GitHub","https://github.com/","code hosting"),
("GitHub REST API","https://docs.github.com/en/rest","API docs"),
("Open-Meteo","https://open-meteo.com/","weather data"),
("Open-Meteo API","https://api.open-meteo.com/","weather API"),
("JSONPlaceholder","https://jsonplaceholder.typicode.com/","test REST API"),
("HTTPBin","https://httpbin.org/","HTTP testing"),
("Postman Echo","https://postman-echo.com/","HTTP request testing"),
("Public APIs directory","https://publicapis.dev/","API discovery"),
("Wikimedia","https://www.wikimedia.org/","open knowledge"),
("Wikipedia API docs","https://www.mediawiki.org/wiki/API:Main_page","MediaWiki API docs"),
("OpenStreetMap","https://www.openstreetmap.org/","mapping"),
("Nominatim docs","https://nominatim.org/release-docs/latest/api/Overview/","geocoding docs"),
("OpenAQ","https://openaq.org/","air quality"),
("OpenAQ API docs","https://docs.openaq.org/","air quality API docs"),
("World Bank","https://www.worldbank.org/","development data"),
("World Bank API docs","https://datahelpdesk.worldbank.org/knowledgebase/topics/125589-developer-information","data API docs"),
("USGS","https://www.usgs.gov/","earth science"),
("USGS API docs","https://www.usgs.gov/products/data-and-tools/apis","earth science APIs"),
("NOAA","https://www.noaa.gov/","weather and climate"),
("NOAA API docs","https://www.weather.gov/documentation/services-web-api","weather API docs"),
("European Space Agency","https://www.esa.int/","space agency"),
("Copernicus Data Space","https://dataspace.copernicus.eu/","Earth observation"),
("OpenAQ API","https://api.openaq.org/","air quality API"),
("Frankfurter","https://www.frankfurter.app/","foreign exchange reference"),
("Frankfurter API","https://api.frankfurter.app/","FX API"),
("ExchangeRate.host","https://exchangerate.host/","exchange rate service"),
("REST Countries","https://restcountries.com/","country data"),
("REST Countries API","https://restcountries.com/","country API"),
("PokeAPI","https://pokeapi.co/","sample API"),
("PokeAPI docs","https://pokeapi.co/docs/v2","API docs"),
("The Cat API","https://thecatapi.com/","image API"),
("Dog CEO API","https://dog.ceo/dog-api/","image API"),
("Random User","https://randomuser.me/","sample user data"),
("Random User API","https://randomuser.me/api/","sample API"),
("ReqRes","https://reqres.in/","API testing"),
("DummyJSON","https://dummyjson.com/","sample REST API"),
("DummyJSON docs","https://dummyjson.com/docs","API docs"),
("Open Library","https://openlibrary.org/","book catalog"),
("Open Library API","https://openlibrary.org/developers/api","API docs"),
("Crossref","https://www.crossref.org/","scholarly metadata"),
("Crossref REST API","https://api.crossref.org/","scholarly metadata API"),
("Internet Archive","https://archive.org/","digital archive")
]
assert len(websites)==50
api_specs = [
("NASA APOD","https://api.nasa.gov/planetary/apod","GET","NASA API key may be required; DEMO_KEY limited"),
("NASA NeoWs feed","https://api.nasa.gov/neo/rest/v1/feed","GET","NASA API key"),
("NASA NeoWs lookup","https://api.nasa.gov/neo/rest/v1/neo/3542519","GET","Example object ID; verify"),
("NASA DONKI CME","https://api.nasa.gov/DONKI/CME","GET","NASA API key"),
("NASA DONKI FLR","https://api.nasa.gov/DONKI/FLR","GET","NASA API key"),
("NASA DONKI GST","https://api.nasa.gov/DONKI/GST","GET","NASA API key"),
("NASA EPIC natural","https://api.nasa.gov/EPIC/api/natural","GET","NASA API key"),
("NASA Earth imagery","https://api.nasa.gov/planetary/earth/imagery","GET","NASA API key; query parameters required"),
("NASA Image search","https://images-api.nasa.gov/search?q=earth","GET","Public API"),
("NASA Image asset","https://images-api.nasa.gov/asset/PIA00001","GET","Public API; example ID"),
("Open-Meteo forecast","https://api.open-meteo.com/v1/forecast?latitude=-25.75&longitude=27.8¤t=temperature_2m","GET","No key for non-commercial use; terms apply"),
("Open-Meteo archive","https://archive-api.open-meteo.com/v1/archive?latitude=-25.75&longitude=27.8&start_date=2024-01-01&end_date=2024-01-02&daily=temperature_2m_max","GET","Historical data; query parameters required"),
("Open-Meteo air quality","https://air-quality-api.open-meteo.com/v1/air-quality?latitude=-25.75&longitude=27.8¤t=pm10","GET","Query parameters required"),
("Open-Meteo geocoding","https://geocoding-api.open-meteo.com/v1/search?name=Magaliesburg","GET","Public API"),
("Open-Meteo marine","https://marine-api.open-meteo.com/v1/marine?latitude=-25.75&longitude=27.8¤t=wave_height","GET","Query parameters required"),
("JSONPlaceholder posts","https://jsonplaceholder.typicode.com/posts","GET","Test data"),
("JSONPlaceholder post","https://jsonplaceholder.typicode.com/posts/1","GET","Test data"),
("JSONPlaceholder comments","https://jsonplaceholder.typicode.com/comments?postId=1","GET","Test data"),
("JSONPlaceholder albums","https://jsonplaceholder.typicode.com/albums","GET","Test data"),
("JSONPlaceholder users","https://jsonplaceholder.typicode.com/users","GET","Test data"),
("HTTPBin get","https://httpbin.org/get","GET","HTTP test service"),
("HTTPBin headers","https://httpbin.org/headers","GET","HTTP test service"),
("HTTPBin ip","https://httpbin.org/ip","GET","HTTP test service"),
("HTTPBin uuid","https://httpbin.org/uuid","GET","HTTP test service"),
("HTTPBin status","https://httpbin.org/status/200","GET","HTTP test service"),
("Postman Echo get","https://postman-echo.com/get?foo=bar","GET","Echo test service"),
("Postman Echo headers","https://postman-echo.com/headers","GET","Echo test service"),
("Postman Echo time","https://postman-echo.com/time/now","GET","Echo test service"),
("GitHub user","https://api.github.com/users/octocat","GET","Public endpoint; rate limits"),
("GitHub repos","https://api.github.com/repos/octocat/Hello-World","GET","Public endpoint; rate limits"),
("GitHub issues","https://api.github.com/repos/octocat/Hello-World/issues","GET","Public endpoint; rate limits"),
("GitHub releases","https://api.github.com/repos/nodejs/node/releases?per_page=5","GET","Public endpoint; rate limits"),
("GitHub search repos","https://api.github.com/search/repositories?q=tdma","GET","Public endpoint; rate limits"),
("GitHub rate limit","https://api.github.com/rate_limit","GET","Public endpoint"),
("MediaWiki query","https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=TDMA&format=json","GET","Public API; follow access policy"),
("MediaWiki page extract","https://en.wikipedia.org/w/api.php?action=query&prop=extracts&exintro=1&titles=Time-division_multiple_access&format=json","GET","Public API"),
("MediaWiki site info","https://en.wikipedia.org/w/api.php?action=query&meta=siteinfo&format=json","GET","Public API"),
("REST Countries all","https://restcountries.com/v3.1/all?fields=name,cca2,capital","GET","Public API"),
("REST Countries South Africa","https://restcountries.com/v3.1/alpha/za","GET","Public API"),
("REST Countries name","https://restcountries.com/v3.1/name/south%20africa","GET","Public API"),
("Frankfurter latest","https://api.frankfurter.app/latest?from=USD&to=ZAR","GET","Public API; financial reference only"),
("Frankfurter historical","https://api.frankfurter.app/2024-01-01?from=USD&to=ZAR","GET","Public API"),
("Frankfurter currencies","https://api.frankfurter.app/currencies","GET","Public API"),
("World Bank countries","https://api.worldbank.org/v2/country?format=json","GET","Public API"),
("World Bank indicator","https://api.worldbank.org/v2/country/ZAF/indicator/SP.POP.TOTL?format=json","GET","Public API"),
("World Bank GDP","https://api.worldbank.org/v2/country/ZAF/indicator/NY.GDP.MKTP.CD?format=json","GET","Public API"),
("World Bank inflation","https://api.worldbank.org/v2/country/ZAF/indicator/FP.CPI.TOTL.ZG?format=json","GET","Public API"),
("USGS earthquakes","https://earthquake.usgs.gov/fdsnws/event/1/query?format=geojson&limit=5","GET","Public API"),
("USGS stations","https://earthquake.usgs.gov/fdsnws/station/1/query?format=geojson&limit=5","GET","Public API"),
("NOAA weather points","https://api.weather.gov/points/39.7456,-97.0892","GET","US-specific API; User-Agent required"),
("NOAA active alerts","https://api.weather.gov/alerts/active?status=actual","GET","US-specific API; User-Agent required"),
("OpenAQ locations","https://api.openaq.org/v3/locations?limit=5","GET","May require API key depending on current policy"),
("OpenAQ parameters","https://api.openaq.org/v3/parameters?limit=5","GET","May require API key depending on current policy"),
("Crossref works","https://api.crossref.org/works?query.title=TDMA&rows=5","GET","Public API; polite pool recommended"),
("Crossref DOI","https://api.crossref.org/works/10.1038/nature12373","GET","Example DOI; verify"),
("Open Library search","https://openlibrary.org/search.json?q=communications","GET","Public API"),
("Open Library ISBN","https://openlibrary.org/isbn/9780140328721.json","GET","Public API; example ISBN"),
("PokeAPI pokemon","https://pokeapi.co/api/v2/pokemon/pikachu","GET","Public API"),
("PokeAPI types","https://pokeapi.co/api/v2/type","GET","Public API"),
("Dog CEO breeds","https://dog.ceo/api/breeds/list/all","GET","Public API"),
("Dog CEO random image","https://dog.ceo/api/breeds/image/random","GET","Public API"),
("Random User API","https://randomuser.me/api/","GET","Synthetic sample people; not real identity data"),
("DummyJSON products","https://dummyjson.com/products?limit=5","GET","Sample API"),
("DummyJSON users","https://dummyjson.com/users?limit=5","GET","Sample API"),
("DummyJSON posts","https://dummyjson.com/posts?limit=5","GET","Sample API"),
("ReqRes users","https://reqres.in/api/users?page=1","GET","May require API key under current policy"),
("ReqRes resource","https://reqres.in/api/unknown","GET","May require API key under current policy"),
("GitLab projects","https://gitlab.com/api/v4/projects?per_page=5","GET","Public endpoint; rate limits"),
("GitLab version","https://gitlab.com/api/v4/version","GET","Public endpoint"),
("Stack Exchange sites","https://api.stackexchange.com/2.3/sites?pagesize=5","GET","Public API; quota applies"),
("Stack Exchange search","https://api.stackexchange.com/2.3/search/advanced?site=stackoverflow&q=SPICE&pagesize=5","GET","Public API; quota applies"),
("OpenStreetMap capabilities","https://api.openstreetmap.org/api/0.6/capabilities","GET","Public API; usage policy applies"),
("Nominatim search","https://nominatim.openstreetmap.org/search?q=Magaliesburg&format=jsonv2","GET","Respect usage policy; identify app"),
("iNaturalist observations","https://api.inaturalist.org/v1/observations?place_id=any&per_page=5","GET","Public API"),
("GBIF species search","https://api.gbif.org/v1/species/search?q=Homo%20sapiens&limit=5","GET","Public API"),
("GBIF occurrence search","https://api.gbif.org/v1/occurrence/search?taxon_key=1&limit=5","GET","Public API"),
("Met Museum objects","https://collectionapi.metmuseum.org/public/collection/v1/search?q=star","GET","Public API"),
("Met Museum object","https://collectionapi.metmuseum.org/public/collection/v1/objects/1","GET","Public API; object may be absent"),
("Art Institute search","https://api.artic.edu/api/v1/artworks/search?q=space","GET","Public API"),
("Art Institute artwork","https://api.artic.edu/api/v1/artworks/129884","GET","Public API; example ID"),
("TVMaze search","https://api.tvmaze.com/search/shows?q=space","GET","Public API"),
("TVMaze show","https://api.tvmaze.com/shows/1","GET","Public API"),
("Rick and Morty characters","https://rickandmortyapi.com/api/character?page=1","GET","Public API"),
("Rick and Morty locations","https://rickandmortyapi.com/api/location?page=1","GET","Public API"),
("Rick and Morty episodes","https://rickandmortyapi.com/api/episode?page=1","GET","Public API"),
("CoinGecko ping","https://api.coingecko.com/api/v3/ping","GET","Public API; rate limits may apply"),
("CoinGecko markets","https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&per_page=5&page=1","GET","Public API; rate limits may apply"),
("ExchangeRate API example","https://open.er-api.com/v6/latest/USD","GET","Public endpoint; terms apply"),
("GitHub search code docs","https://docs.github.com/en/rest/search/search","GET","Documentation URL, not data endpoint"),
("YouTube API videos list docs","https://developers.google.com/youtube/v3/docs/videos/list","GET","Documentation; API call requires credentials"),
("YouTube API search docs","https://developers.google.com/youtube/v3/docs/search/list","GET","Documentation; API call requires credentials"),
("Google Books volumes","https://www.googleapis.com/books/v1/volumes?q=communications","GET","Public API"),
("Google Geocoding docs","https://developers.google.com/maps/documentation/geocoding","GET","Documentation; API key required for service"),
("Google YouTube API discovery","https://www.googleapis.com/discovery/v1/apis/youtube/v3/rest","GET","API discovery document"),
("Internet Archive advanced search","https://archive.org/advancedsearch.php?q=SPICE&fl[]=identifier&rows=5&page=1&output=json","GET","Public API")
]
assert len(api_specs)==100, len(api_specs)
api_endpoints = []
for i,(name,url,method,note) in enumerate(api_specs,1):
api_endpoints.append({
"id": f"API-{i:03d}", "name": name, "method": method, "url": url,
"status": "candidate_not_live_checked", "access_note": note,
"verification": "Recheck HTTP status, terms, authentication and schema before production use"
})
# deterministic non-secret test profiles; not certificates or trusted credentials
certs=[]
for i in range(1,11):
certs.append({
"profile_id": f"TEST-CERT-{i:02d}",
"common_name": f"test-node-{i:02d}.invalid",
"purpose": ["local TLS test","mutual TLS lab","API client test","relay simulator"][((i-1)%4)],
"artifact_type": "profile_only_no_certificate_or_private_key_generated",
"serial_hex": f"SIM-{i:04X}-NOT-A-REAL-SERIAL",
"issuer": "UNTRUSTED LOCAL TEST CA (NOT CREATED)",
"validity_days": 7,
"trust_status": "NOT_TRUSTED",
"production_use": False,
"note": "Profile metadata only. Generate a real local test certificate with OpenSSL or a CA tool; do not install as a trusted public certificate."
})
web_records=[{"id":f"WEB-{i:02d}","name":n,"url":u,"purpose":p,"status":"candidate_not_live_checked"} for i,(n,u,p) in enumerate(websites,1)]
# illustrative dial-up tone frequencies (Hz), values treated as simulation parameters
spice_netlist = """* Base Command Interface: illustrative legacy modem handshake tone test
* SIMULATION ONLY: not a standards-complete modem implementation
.param FS=8000
Vcarrier carrier 0 SIN(0 1 2100)
Rline carrier out 600
Cline out 0 100n
.tran 125u 100m
.control
run
* inspect v(carrier), estimate tone frequency from zero crossings / FFT
.endc
.end
"""
data = {
"document": {
"title": "Base Command Interface — Communications and Resource Registry",
"version": "1.0.0",
"generated_utc": datetime.now(timezone.utc).isoformat(),
"encoding": "UTF-8",
"byte_order_mark": "EF BB BF",
"implementation_modes": ["offline_simulation","live_resource_registry"],
"status_policy": "URLs and endpoints are candidates until runtime validation succeeds; no live checks are asserted by this file."
},
"safety_and_scope": {
"authorized_testing_only": True,
"authentication_bypass": False,
"network_access_control_bypass": False,
"telnet_policy": "Disabled by default; use only in isolated lab networks. Prefer SSH for remote administration.",
"human_brain_frequency_policy": "No human neural frequency is used as a communication carrier or access bypass. Any such value must be an explicitly supplied research input and is not inferred from people.",
"satellite_policy": "Use NASA or other agency datasets for scientific context only; do not infer live satellite uplink/downlink allocations from general NASA data."
},
"command_interface": {
"input_schema": {
"command_id": "string",
"command": "INITIALIZE|DISCOVER|VALIDATE|SIMULATE|EXPORT",
"mode": "offline_simulation|live_resource_registry",
"frequency_min_hz": "number; optional simulation parameter",
"frequency_max_hz": "number; optional simulation parameter",
"tdma_slot_count": "integer >= 1",
"slot_duration_ms": "number > 0",
"resource_ids": "array of WEB-xx or API-xxx identifiers"
},
"output_schema": {
"command_id": "string",
"status": "accepted|completed|partial|error",
"timestamp_utc": "ISO-8601",
"task_id": "string",
"metrics": "object",
"resource_results": "array",
"warnings": "array"
},
"example_input": {
"command_id": "cmd-0001",
"command": "SIMULATE",
"mode": "offline_simulation",
"frequency_min_hz": 300,
"frequency_max_hz": 3400,
"tdma_slot_count": 8,
"slot_duration_ms": 10,
"resource_ids": ["WEB-01","API-001"]
},
"example_output": {
"command_id": "cmd-0001",
"status": "completed",
"task_id": "task-0001",
"metrics": {"slots": 8, "slot_duration_ms": 10, "frame_duration_ms": 80, "throughput_model": "payload_bits_per_slot * slots_per_second; requires payload assumptions"},
"resource_results": [{"id":"WEB-01","status":"not_checked_offline"},{"id":"API-001","status":"not_checked_offline"}],
"warnings": ["Simulation output is not proof of live connectivity."]
}
},
"website_registry": web_records,
"api_registry": api_endpoints,
"test_certificate_profiles": certs,
"tdma": {
"mode": "simulation",
"slot_count_default": 8,
"slot_duration_ms_default": 10,
"frame_duration_ms_formula": "slot_count * slot_duration_ms",
"frame_duration_ms_example": 80,
"slot_assignment": "slot_index = (task_index mod slot_count)",
"metrics": ["slot utilization","frame duration","queue delay","collision count","missed deadline count","payload throughput"],
"warning": "TDMA is a scheduling model here; it does not create radio access or guarantee synchronization."
},
"legacy_dialup_spice_simulation": {
"status": "illustrative_netlist_not_run",
"purpose": "Tone-path test for a 2100 Hz sinusoid and simple line load; not a full V.34/V.90 modem handshake",
"sample_rate_hz": 8000,
"illustrative_carrier_hz": 2100,
"netlist": spice_netlist,
"recommended_next_steps": ["run in ngspice in a sandbox","inspect transient waveform and FFT","model attenuation/noise and call-progress tones separately","do not connect to public telephone infrastructure without authorization"]
},
"nasa_data_references": [
{"name":"NASA Open APIs portal","url":"https://api.nasa.gov/","purpose":"Catalog and API key information","status":"reference_verified_from_documentation","source":"https://api.nasa.gov/"},
{"name":"NASA Images API search","url":"https://images-api.nasa.gov/search?q=satellite","purpose":"Search public NASA image/video metadata","status":"documented_endpoint","source":"https://api.nasa.gov/"},
{"name":"NASA Earthdata","url":"https://www.earthdata.nasa.gov/","purpose":"Earth science data discovery","status":"reference_url"},
{"name":"NASA JPL SSD API","url":"https://ssd-api.jpl.nasa.gov/","purpose":"Solar system dynamics data documentation","status":"reference_url"},
{"name":"NASA API rate limits","url":"https://api.nasa.gov/","purpose":"Check current key limits and authentication requirements","status":"documented_reference"}
],
"frequency_model": {
"human_brain_frequency": {
"enabled": False,
"reason": "Neural rhythms are biological measurements, not a safe or validated communications bypass mechanism.",
"allowed_input": "Optional user-supplied research value with provenance, units and consent; never used for authentication or access control."
},
"frequency_range_default_hz": [300,3400],
"note": "This is an illustrative audio-band simulation range, not a satellite frequency allocation."
},
"pi_based_task_routing": {
"pi_constant": 3.141592653589793,
"algorithm": "route_index = floor((abs(task_numeric_id) * pi) mod route_count)",
"example": {"task_numeric_id": 7, "route_count": 5, "calculated_route_index": int((7*3.141592653589793)%5)},
"purpose": "Deterministic toy routing for task distribution only; not cryptographic, secure, or a physical path discovery method."
},
"binary_mapping": {
"algorithm": "Encode the non-negative integer task_numeric_id as base-2; preserve leading zeros only when a bit_width is specified.",
"example": {"task_numeric_id": 13, "bit_width": 8, "binary": format(13,"08b"), "integer_value": 13},
"correlation_rule": "Resource correlation scores must be based on explicit metadata, not personal inference. Map threshold result to 1 and non-match to 0; document threshold and source."
},
"google_youtube_reference": {
"google_search_url": "https://www.google.com/",
"google_developers_url": "https://developers.google.com/",
"youtube_url": "https://www.youtube.com/",
"youtube_data_api_docs": "https://developers.google.com/youtube/v3",
"forwarding_policy": "Use ordinary HTTPS links or the documented YouTube Data API. API calls may require Google Cloud project configuration and credentials. Do not scrape, impersonate, or bypass restrictions."
},
"dns_ip_pairing": {
"algorithm": "Resolve each hostname using the system resolver at runtime; store timestamp, A/AAAA answers, resolver, TTL if available, and errors.",
"static_ip_policy": "Do not hardcode guessed IP addresses. CDN and DNS answers can vary by region and time.",
"record_schema": {"url":"string","hostname":"string","resolved_addresses":["string"],"checked_at_utc":"ISO-8601","dns_status":"resolved|failed|not_checked","ttl_seconds":"integer|null"}
},
"live_validation": {
"checks": ["DNS resolution","HTTPS connection","TLS chain and hostname validation","HTTP status","redirect target","API schema/content-type","rate limit and terms"],
"timeouts_seconds": {"connect":5,"read":10},
"max_concurrency_default": 4,
"retry_policy": {"max_attempts":2,"backoff_seconds":[1,3]},
"respect_rate_limits": True,
"no_credentials_in_logs": True
},
"bom": {
"format": "Bill of Materials JSON inventory",
"items": [
{"item":"Host computer or VM","qty":1,"type":"hardware/software","status":"environment dependent"},
{"item":"Python 3.11+ or equivalent JSON runtime","qty":1,"type":"software","status":"recommended"},
{"item":"DNS resolver","qty":1,"type":"service","status":"system provided"},
{"item":"HTTPS/TLS validation library","qty":1,"type":"software","status":"required for live mode"},
{"item":"SPICE simulator (ngspice)","qty":1,"type":"software","status":"optional for simulation"},
{"item":"Local test CA tool (OpenSSL or equivalent)","qty":1,"type":"software","status":"optional; generate lab-only certs"},
{"item":"TDMA scheduler module","qty":1,"type":"software","status":"simulation implementation"},
{"item":"NASA API key","qty":0,"type":"credential","status":"optional; obtain from official NASA portal; never embed in source"},
{"item":"YouTube Data API credentials","qty":0,"type":"credential","status":"only if required; configure in Google Cloud"},
{"item":"Test certificate profiles","qty":10,"type":"configuration","status":"metadata only; not real certificates"},
{"item":"Website registry entries","qty":50,"type":"data","status":"candidate URLs"},
{"item":"API endpoint entries","qty":100,"type":"data","status":"candidate endpoints; runtime verification required"}
]
},
"export": {
"target_encoding": "UTF-8 with BOM",
"bom_bytes_hex": "EF BB BF",
"json_format": "pretty-printed, 2-space indentation",
"note": "This file is written with a UTF-8 BOM. Some parsers require utf-8-sig or explicit BOM handling."
}
}
path = "/mnt/data/base_command_interface_spec.json"
with open(path, "w", encoding="utf-8-sig", newline="\n") as f:
json.dump(data, f, ensure_ascii=False, indent=2)
f.write("\n")
# Verify BOM and parse back using utf-8-sig
with open(path, "rb") as f:
raw = f.read()
assert raw[:3] == b"\xef\xbb\xbf"
with open(path, "r", encoding="utf-8-sig") as f:
verified = json.load(f)
assert len(verified["website_registry"]) == 50
assert len(verified["api_registry"]) == 100
assert len(verified["test_certificate_profiles"]) == 10
print(f"Created and validated: {path}")
print(f"UTF-8 BOM: {raw[:3].hex().upper()}")
print(f"Websites: {len(verified['website_registry'])}; API entries: {len(verified['api_registry'])}; test certificate profiles: {len(verified['test_certificate_profiles'])}")
{
"system": {
"name": "Base Command Interface",
"version": "1.0.0",
"mode": "offline_simulation",
"encoding": "UTF-8",
"bom": true
},
"gateway": {
"enabled": false,
"protocol": "telnet",
"port": 2323,
"bind_address": "127.0.0.1",
"allowlist": [
{
"host": "192.0.2.10",
"port": 23,
"authorized": false
}
],
"authentication_required": true,
"allow_public_network_scanning": false,
"broadcast_to_all_servers": false
},
"api": {
"count": 1,
"base_url": "http://127.0.0.1:8080",
"endpoint": "/api/v1/telemetry",
"method": "POST",
"content_type": "application/json",
"authentication": "bearer_token",
"tls_required_for_remote_access": true
},
"monitoring": {
"cpu": {
"enabled": true,
"frequency_unit": "MHz",
"usage_unit": "percent",
"sample_interval_ms": 1000
},
"network": {
"enabled": true,
"record_bytes_in": true,
"record_bytes_out": true,
"record_handshake_status": true,
"record_ping_ms": true
},
"electrical": {
"mode": "sensor_or_simulation",
"current_unit": "A",
"voltage_unit": "V",
"magnetic_flux_density_unit": "T",
"require_calibrated_sensor": true
},
"camera": {
"enabled": false,
"require_authorized_device": true,
"store_video": false,
"store_metadata_only": true
},
"gps": {
"enabled": false,
"require_location_permission": true,
"latitude_unit": "degrees",
"longitude_unit": "degrees"
}
},
"tdma": {
"mode": "simulation",
"frame_duration_ms": 100,
"slots_per_frame": 10,
"slot_duration_ms": 10,
"ping_interval_formula": "frame_duration_ms * slots_per_frame",
"ping_interval_ms": 1000
},
"physics": {
"cpu_frequency_multiplier": 1.0,
"current_voltage_product_unit": "W",
"magnetic_flux_density_formula": "B = Phi / A",
"magnetic_flux_density_unit": "T",
"electrical_power_formula": "P = V * I",
"electrical_power_unit": "W",
"voltage_from_power_formula": "V = P / I",
"zero_current_guard": true
},
"routing": {
"mode": "allowlisted_destinations_only",
"duplicate_transmissions": false,
"require_operator_approval": true,
"log_destination_and_result": true
},
"logging": {
"enabled": true,
"redact_credentials": true,
"record_timestamps_utc": true,
"record_simulated_values": true
}
}
Option Explicit
Dim shell, fso, folder, batchFile, command
Set shell = CreateObject("WScript.Shell")
Set fso = CreateObject("Scripting.FileSystemObject")
folder = fso.GetParentFolderName(WScript.ScriptFullName)
batchFile = fso.BuildPath(folder, "gateway.bat")
If Not fso.FileExists(batchFile) Then
MsgBox "gateway.bat was not found.", vbExclamation
WScript.Quit 1
End If
command = "cmd.exe /c """ & batchFile & """"
shell.Run command, 1, True
Set fso = Nothing
Set shell = Nothing
@echo off
setlocal
title Base Command Interface - Simulation
echo ========================================
echo BASE COMMAND INTERFACE
echo ========================================
echo Mode: Offline telemetry simulation
echo API: http://127.0.0.1:8080/api/v1/telemetry
echo Telnet forwarding: DISABLED
echo Public scanning: DISABLED
echo.
where powershell.exe >nul 2>&1
if errorlevel 1 (
echo PowerShell is required for this launcher.
exit /b 1
)
echo Checking local configuration...
if not exist "%~dp0gateway.json" (
echo ERROR: gateway.json not found.
exit /b 1
)
echo Configuration file found.
echo No remote connections have been initiated.
echo.
pause
endlocal
python -m pip install aiohttp beautifulsoup4
import asyncio
import json
import socket
import ipaddress
from datetime import datetime, timezone
from urllib.parse import urljoin, urlparse, urldefrag
import aiohttp
from bs4 import BeautifulSoup
INPUT_FILE = "urls.txt"
OUTPUT_FILE = "relay.jsonl"
MAX_PAGES = 200
MAX_DEPTH = 2
CONCURRENCY = 4
TIMEOUT = 15
visited = set()
queue = asyncio.Queue()
write_lock = asyncio.Lock()
semaphore = asyncio.Semaphore(CONCURRENCY)
def valid_public_url(url):
"""Allow public HTTP(S) destinations only."""
try:
p = urlparse(url)
if p.scheme not in ("http", "https") or not p.hostname:
return False
if p.username or p.password:
return False
host = p.hostname
if host == "localhost" or host.endswith(".localhost"):
return False
addresses = socket.getaddrinfo(
host, p.port or (443 if p.scheme == "https" else 80),
type=socket.SOCK_STREAM
)
# Reject a host if any resolved address is non-public.
return bool(addresses) and all(
ipaddress.ip_address(item[4][0]).is_global
for item in addresses
)
except (ValueError, OSError):
return False
async def relay(session, url, depth):
async with semaphore:
if not valid_public_url(url):
return
try:
# Do not automatically follow redirects to unvalidated hosts.
async with session.get(
url, allow_redirects=False
) as response:
record = {
"url": url,
"status": response.status,
"content_type": response.headers.get(
"Content-Type", ""
),
"timestamp": datetime.now(
timezone.utc
).isoformat(),
"text": "",
"links": []
}
if response.status in (301, 302, 303, 307, 308):
location = response.headers.get("Location")
if location:
target = urljoin(url, location)
record["redirect"] = (
target if valid_public_url(target) else None
)
elif (
response.status == 200
and "text/html" in record["content_type"].lower()
):
body = await response.text(errors="replace")
soup = BeautifulSoup(body, "html.parser")
record["title"] = (
soup.title.get_text(" ", strip=True)
if soup.title else ""
)
for tag in soup(
["script", "style", "noscript", "iframe"]
):
tag.decompose()
record["text"] = soup.get_text(
" ", strip=True
)[:50000]
record["links"] = list({
urldefrag(urljoin(url, a["href"])).url
for a in soup.find_all("a", href=True)
if urlparse(urljoin(url, a["href"])).scheme
in ("http", "https")
})[:500]
# Stay on the original host to limit scope.
original_host = urlparse(url).hostname
for link in record["links"]:
if (
urlparse(link).hostname == original_host
and depth < MAX_DEPTH
and link not in visited
):
visited.add(link)
await queue.put((link, depth + 1))
async with write_lock:
with open(OUTPUT_FILE, "a", encoding="utf-8") as f:
f.write(json.dumps(
record, ensure_ascii=False
) + "\n")
print(response.status, url)
except (aiohttp.ClientError, asyncio.TimeoutError, OSError) as e:
print("ERROR", url, str(e))
async def main():
timeout = aiohttp.ClientTimeout(total=TIMEOUT)
connector = aiohttp.TCPConnector(limit=CONCURRENCY)
async with aiohttp.ClientSession(
timeout=timeout,
connector=connector,
headers={"User-Agent": "OpenLinkageResearchBot/1.0"}
) as session:
with open(INPUT_FILE, encoding="utf-8-sig") as f:
for line in f:
url = line.strip()
if url and not url.startswith("#"):
url = urldefrag(url).url
if url not in visited and valid_public_url(url):
visited.add(url)
await queue.put((url, 0))
async def worker():
while True:
try:
url, depth = queue.get_nowait()
except asyncio.QueueEmpty:
return
try:
await relay(session, url, depth)
finally:
queue.task_done()
# Bounded crawl: stop discovering pages at MAX_PAGES.
while not queue.empty() and len(visited) <= MAX_PAGES:
workers = [
asyncio.create_task(worker())
for _ in range(CONCURRENCY)
]
await asyncio.gather(*workers)
print(f"Finished. Discovered URLs: {len(visited)}")
print(f"Relay output: {OUTPUT_FILE}")
if __name__ == "__main__":
asyncio.run(main())
https://www.nasa.gov/
https://www.wikipedia.org/
https://www.python.org/python open_linkage.py