HackmanClub/HackmanCMS
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: HackmanClub:df11e32e70ca3abebccf948a89f7868e33d5d3c8
Branches Tags
HackmanClub:main
...
compare: HackmanClub:f6e768c246f23afab15956784a9b6e9932597a3a
Branches Tags
HackmanClub:main

2 commits
df11e32e70 ... f6e768c246

Author SHA1 Message Date
Bashy
f6e768c246 Expand bot/automated-tool filter in site log importer 
Add empty UA early return and a comprehensive list of bot signals covering
HTTP libraries, SEO platforms, uptime monitors, security scanners, AI
crawlers, and generic scanner vocabulary.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
 2026-05-27 19:29:29 +03:00
Bashy
dfca6df629 Send Discord snowflake IDs as strings to prevent JS precision loss 
channel_id and role_id are 64-bit integers that exceed Number.MAX_SAFE_INTEGER,
causing silent rounding in the browser. Cast to string in the GET response;
PHP stores them back as integers on save (64-bit PHP handles them correctly).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
 2026-05-27 19:29:23 +03:00
2 changed files with 57 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

52
bin/import-site-logs.php
View file

@ -280,9 +280,57 @@ function isInteresting(array $row, string $pathPrefix): bool {
'mp3','wav','ogg','pdf','zip','tar','gz','xml','txt'];
if (in_array($ext, $asset, true)) return false;
// Skip obvious bots
// Empty UA — almost certainly automated
$ua = strtolower((string)$row['ua']);
foreach (['bot','crawl','spider','curl','wget','headless','scrapy','go-http','python-requests','okhttp'] as $needle) {
if ($ua === '') return false;
// Comprehensive bot / automated-tool filter
static $botSignals = [
// Generic signals — catches most search engines + named bots
'bot', 'crawl', 'spider', 'headless', 'scrapy', 'slurp',
// HTTP tools and libraries
'curl', 'wget',
'python-requests', 'python-urllib', 'python-httpx',
'go-http', 'okhttp',
'libwww-perl', 'lwp/',
'guzzlehttp', 'guzzle/',
'java/', 'apache-httpclient',
'node-fetch', 'undici/', 'got/', 'axios/',
'aiohttp/', 'httpx/',
'mechanize', 'faraday/', 'httpie', 'ruby',
// SEO / link-analysis platforms
'ahrefs', 'semrush', 'moz.com', 'sitechecker', 'similarweb',
'screaming frog', 'dataforseo', 'serpstat', 'seokicks', 'babbar',
'dotbot', 'duckduckgo-favicons',
// Uptime / monitoring services
'uptimerobot', 'pingdom', 'statuscake', 'site24x7', 'freshping',
'betteruptime', 'hetrixtools', 'nodeping', 'hyperping',
'updown.io', 'check_http', 'healthcheck',
// Security / vulnerability scanners
'nikto', 'nessus', 'masscan', 'zgrab', 'nuclei',
'sqlmap', 'whatweb', 'wfuzz', 'dirbuster', 'gobuster',
'feroxbuster', 'burp',
// Social link-preview fetchers (not caught by "bot")
'whatsapp', 'skypeuripreview', 'slack-imgproxy', 'vkshare',
// Archive / site copiers
'ia_archiver', 'httrack',
// APM / observability agents
'datadog', 'newrelic', 'dynatrace',
// AI crawlers not containing "bot"
'cohere-ai', 'anthropic-ai', 'meta-externalagent',
// Generic scanner vocabulary
'scanner', 'checker', 'prober', 'monitor',
];
foreach ($botSignals as $needle) {
if (str_contains($ua, $needle)) return false;
}
return true;

7
web/api/botconfig.php
View file

@ -39,6 +39,13 @@ if ($method === 'GET') {
$safe = $config;
if (isset($safe['linkedin']['access_token'])) $safe['linkedin']['access_token'] = $safe['linkedin']['access_token'] ? '***' : null;
if (isset($safe['linkedin']['refresh_token'])) $safe['linkedin']['refresh_token'] = $safe['linkedin']['refresh_token'] ? '***' : null;
// Discord snowflake IDs exceed JS Number.MAX_SAFE_INTEGER — send as strings
foreach (['twitch', 'rss'] as $section) {
foreach ($safe[$section] ?? [] as $key => $item) {
if (isset($item['channel_id'])) $safe[$section][$key]['channel_id'] = (string)$item['channel_id'];
if (isset($item['role_id']) && $item['role_id'] !== null) $safe[$section][$key]['role_id'] = (string)$item['role_id'];
}
}
echo json_encode(['config' => $safe]);
exit;
}