移除临时脚本;脱敏
This commit is contained in:
parent
3d63eb4e96
commit
d8292f9bca
|
|
@ -0,0 +1 @@
|
||||||
|
.env
|
||||||
|
|
@ -1,49 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
import asyncio
|
|
||||||
from playwright.async_api import async_playwright
|
|
||||||
|
|
||||||
TARGET_PAGE = "https://gd.10086.cn/gdshop/qdxsd/index.html#/gd-fls/marketingActivity/index?id=1956241557401346048"
|
|
||||||
TARGET_API = "https://gd.10086.cn/gdshop/apigw/adv/ad/getInsertCode"
|
|
||||||
FINGERPRINT_HEADER = "x-device-fingerprint"
|
|
||||||
|
|
||||||
|
|
||||||
async def capture_fingerprint():
|
|
||||||
async with async_playwright() as p:
|
|
||||||
browser = await p.chromium.launch(headless=True)
|
|
||||||
page = await browser.new_page()
|
|
||||||
|
|
||||||
fingerprint_future = asyncio.Future()
|
|
||||||
|
|
||||||
async def handle_request(request):
|
|
||||||
if TARGET_API in request.url and not fingerprint_future.done():
|
|
||||||
headers = request.headers
|
|
||||||
if FINGERPRINT_HEADER in headers:
|
|
||||||
fingerprint = headers[FINGERPRINT_HEADER]
|
|
||||||
fingerprint_future.set_result(fingerprint)
|
|
||||||
|
|
||||||
page.on("request", handle_request)
|
|
||||||
|
|
||||||
print(f"[信息] 正在打开页面...")
|
|
||||||
await page.goto(TARGET_PAGE, wait_until="networkidle")
|
|
||||||
|
|
||||||
try:
|
|
||||||
print(f"[信息] 等待指纹参数...")
|
|
||||||
fingerprint = await asyncio.wait_for(fingerprint_future, timeout=15)
|
|
||||||
print(f"\n{'='*60}")
|
|
||||||
print(f"[成功] 指纹参数捕获成功:")
|
|
||||||
print(f" {FINGERPRINT_HEADER}: {fingerprint}")
|
|
||||||
print(f"{'='*60}\n")
|
|
||||||
return fingerprint
|
|
||||||
except asyncio.TimeoutError:
|
|
||||||
print(f"\n{'='*60}")
|
|
||||||
print(f"[错误] 等待超时,未能捕获到 {FINGERPRINT_HEADER} 参数")
|
|
||||||
print(f"{'='*60}\n")
|
|
||||||
return None
|
|
||||||
finally:
|
|
||||||
await browser.close()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(capture_fingerprint())
|
|
||||||
|
|
||||||
|
|
@ -2,29 +2,39 @@
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
IMAGE_NAME="gd10086-fingerprint-service"
|
IMAGE_NAME="web-signature-service"
|
||||||
CONTAINER_NAME="gd10086-fingerprint-service"
|
CONTAINER_NAME="web-signature-service"
|
||||||
PORT="10086"
|
PORT="8000"
|
||||||
|
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
echo "广东移动指纹获取服务 - 部署脚本"
|
echo "Web Header Signature Service - Deploy"
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
|
|
||||||
cd "$(dirname "$0")"
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
# 检查 .env 文件,如果存在则加载
|
||||||
|
ENV_FILE=""
|
||||||
|
if [ -f ".env" ]; then
|
||||||
|
echo "Found .env file, will use it for deployment."
|
||||||
|
ENV_FILE="--env-file .env"
|
||||||
|
else
|
||||||
|
echo "Warning: .env file not found. Service will use default target configurations."
|
||||||
|
fi
|
||||||
|
|
||||||
case "${1:-up}" in
|
case "${1:-up}" in
|
||||||
up)
|
up)
|
||||||
echo ""
|
echo ""
|
||||||
echo "[1/3] 构建 Docker 镜像..."
|
echo "[1/3] Building Docker image..."
|
||||||
docker build -t ${IMAGE_NAME} .
|
docker build -t ${IMAGE_NAME} .
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "[2/3] 停止旧容器(如果存在)..."
|
echo "[2/3] Stopping old container (if exists)..."
|
||||||
docker stop ${CONTAINER_NAME} 2>/dev/null || true
|
docker stop ${CONTAINER_NAME} 2>/dev/null || true
|
||||||
docker rm ${CONTAINER_NAME} 2>/dev/null || true
|
docker rm ${CONTAINER_NAME} 2>/dev/null || true
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "[3/3] 启动服务..."
|
echo "[3/3] Starting service..."
|
||||||
|
# 注意: 传入环境变量配置
|
||||||
docker run -d \
|
docker run -d \
|
||||||
--name ${CONTAINER_NAME} \
|
--name ${CONTAINER_NAME} \
|
||||||
-p ${PORT}:8000 \
|
-p ${PORT}:8000 \
|
||||||
|
|
@ -32,58 +42,44 @@ case "${1:-up}" in
|
||||||
-v /dev/shm:/dev/shm \
|
-v /dev/shm:/dev/shm \
|
||||||
--security-opt seccomp=unconfined \
|
--security-opt seccomp=unconfined \
|
||||||
-e TZ=Asia/Shanghai \
|
-e TZ=Asia/Shanghai \
|
||||||
|
${ENV_FILE} \
|
||||||
${IMAGE_NAME}
|
${IMAGE_NAME}
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "等待服务启动..."
|
echo "Waiting for service to start..."
|
||||||
sleep 5
|
sleep 3
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
echo "部署完成!"
|
echo "Deployment completed!"
|
||||||
echo ""
|
echo ""
|
||||||
echo "服务地址: http://localhost:${PORT}"
|
echo "Service URL: http://localhost:${PORT}"
|
||||||
echo "API 文档: http://localhost:${PORT}/docs"
|
echo "API Docs: http://localhost:${PORT}/docs"
|
||||||
echo "健康检查: http://localhost:${PORT}/health"
|
echo "Health: http://localhost:${PORT}/health"
|
||||||
echo "获取指纹: http://localhost:${PORT}/fingerprint"
|
echo "Get Sign: http://localhost:${PORT}/fingerprint"
|
||||||
echo ""
|
echo ""
|
||||||
echo "查看日志: docker logs -f ${CONTAINER_NAME}"
|
echo "View Logs: docker logs -f ${CONTAINER_NAME}"
|
||||||
echo "停止服务: docker stop ${CONTAINER_NAME}"
|
echo "Stop: docker stop ${CONTAINER_NAME}"
|
||||||
echo "删除容器: docker rm ${CONTAINER_NAME}"
|
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
;;
|
;;
|
||||||
|
|
||||||
down)
|
down)
|
||||||
echo ""
|
echo "Stopping service..."
|
||||||
echo "停止服务..."
|
|
||||||
docker stop ${CONTAINER_NAME} 2>/dev/null || true
|
docker stop ${CONTAINER_NAME} 2>/dev/null || true
|
||||||
docker rm ${CONTAINER_NAME} 2>/dev/null || true
|
docker rm ${CONTAINER_NAME} 2>/dev/null || true
|
||||||
echo "服务已停止"
|
echo "Service stopped."
|
||||||
;;
|
;;
|
||||||
|
|
||||||
logs)
|
logs)
|
||||||
echo ""
|
|
||||||
echo "查看日志..."
|
|
||||||
docker logs -f ${CONTAINER_NAME}
|
docker logs -f ${CONTAINER_NAME}
|
||||||
;;
|
;;
|
||||||
|
|
||||||
restart)
|
|
||||||
echo ""
|
|
||||||
echo "重启服务..."
|
|
||||||
docker restart ${CONTAINER_NAME}
|
|
||||||
echo "服务已重启"
|
|
||||||
;;
|
|
||||||
|
|
||||||
*)
|
*)
|
||||||
echo ""
|
echo "Usage: $0 [command]"
|
||||||
echo "用法: $0 [命令]"
|
echo "Commands:"
|
||||||
echo ""
|
echo " up - Build and start (default)"
|
||||||
echo "命令:"
|
echo " down - Stop and remove"
|
||||||
echo " up - 构建并启动服务(默认)"
|
echo " logs - View logs"
|
||||||
echo " down - 停止并删除容器"
|
|
||||||
echo " logs - 查看日志"
|
|
||||||
echo " restart - 重启服务"
|
|
||||||
echo ""
|
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|
|
||||||
48
main.py
48
main.py
|
|
@ -1,14 +1,23 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import os
|
||||||
|
import logging
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from fastapi import FastAPI, HTTPException
|
from fastapi import FastAPI, HTTPException
|
||||||
from playwright.async_api import async_playwright
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
TARGET_PAGE = "https://gd.10086.cn/gdshop/qdxsd/index.html#/gd-fls/marketingActivity/index?id=1956241557401346048"
|
# 配置日志
|
||||||
TARGET_API = "https://gd.10086.cn/gdshop/apigw/adv/ad/getInsertCode"
|
logging.basicConfig(
|
||||||
FINGERPRINT_HEADER = "x-device-fingerprint"
|
level=logging.INFO,
|
||||||
|
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||||
|
)
|
||||||
|
logger = logging.getLogger("fingerprint_service")
|
||||||
|
|
||||||
|
TARGET_PAGE = os.getenv("TARGET_PAGE")
|
||||||
|
TARGET_API = os.getenv("TARGET_API")
|
||||||
|
FINGERPRINT_HEADER = os.getenv("FINGERPRINT_HEADER")
|
||||||
|
|
||||||
playwright_instance = None
|
playwright_instance = None
|
||||||
browser = None
|
browser = None
|
||||||
|
|
@ -19,6 +28,7 @@ RESOURCE_CACHE = {}
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
global playwright_instance, browser
|
global playwright_instance, browser
|
||||||
|
logger.info("Starting Playwright instance...")
|
||||||
playwright_instance = await async_playwright().start()
|
playwright_instance = await async_playwright().start()
|
||||||
browser = await playwright_instance.chromium.launch(
|
browser = await playwright_instance.chromium.launch(
|
||||||
headless=True,
|
headless=True,
|
||||||
|
|
@ -29,20 +39,24 @@ async def lifespan(app: FastAPI):
|
||||||
"--disable-software-rasterizer",
|
"--disable-software-rasterizer",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
logger.info("Playwright browser launched successfully.")
|
||||||
yield
|
yield
|
||||||
|
logger.info("Closing Playwright browser...")
|
||||||
await browser.close()
|
await browser.close()
|
||||||
await playwright_instance.stop()
|
await playwright_instance.stop()
|
||||||
|
logger.info("Playwright instance stopped.")
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="广东移动指纹获取服务",
|
title="Web Header Signature Service",
|
||||||
description="获取 x-device-fingerprint 参数的 API 服务",
|
description="API service for dynamic header signature generation via headless browser",
|
||||||
version="1.0.0",
|
version="1.0.0",
|
||||||
lifespan=lifespan
|
lifespan=lifespan
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def get_fingerprint():
|
async def get_fingerprint():
|
||||||
|
logger.info("Opening new browser page...")
|
||||||
page = await browser.new_page()
|
page = await browser.new_page()
|
||||||
try:
|
try:
|
||||||
# [新增] 拦截无用资源,极大提升加载速度
|
# [新增] 拦截无用资源,极大提升加载速度
|
||||||
|
|
@ -59,6 +73,7 @@ async def get_fingerprint():
|
||||||
# 2. 强缓存 JS 文件
|
# 2. 强缓存 JS 文件
|
||||||
if resource_type == "script":
|
if resource_type == "script":
|
||||||
if url in RESOURCE_CACHE:
|
if url in RESOURCE_CACHE:
|
||||||
|
logger.debug(f"Cache hit for script: {url}")
|
||||||
await route.fulfill(
|
await route.fulfill(
|
||||||
status=200,
|
status=200,
|
||||||
headers=RESOURCE_CACHE[url]["headers"],
|
headers=RESOURCE_CACHE[url]["headers"],
|
||||||
|
|
@ -68,6 +83,7 @@ async def get_fingerprint():
|
||||||
|
|
||||||
# 缓存未命中,去真实抓取
|
# 缓存未命中,去真实抓取
|
||||||
try:
|
try:
|
||||||
|
logger.debug(f"Fetching script: {url}")
|
||||||
response = await route.fetch()
|
response = await route.fetch()
|
||||||
body = await response.body()
|
body = await response.body()
|
||||||
RESOURCE_CACHE[url] = {
|
RESOURCE_CACHE[url] = {
|
||||||
|
|
@ -76,7 +92,8 @@ async def get_fingerprint():
|
||||||
}
|
}
|
||||||
await route.fulfill(response=response, body=body)
|
await route.fulfill(response=response, body=body)
|
||||||
return
|
return
|
||||||
except Exception:
|
except Exception as e:
|
||||||
|
logger.warning(f"Failed to fetch script {url}: {e}")
|
||||||
pass # 抓取失败,降级给底层处理
|
pass # 抓取失败,降级给底层处理
|
||||||
|
|
||||||
await route.continue_()
|
await route.continue_()
|
||||||
|
|
@ -89,24 +106,33 @@ async def get_fingerprint():
|
||||||
if TARGET_API in request.url and not fingerprint_future.done():
|
if TARGET_API in request.url and not fingerprint_future.done():
|
||||||
headers = request.headers
|
headers = request.headers
|
||||||
if FINGERPRINT_HEADER in headers:
|
if FINGERPRINT_HEADER in headers:
|
||||||
|
logger.info("Successfully intercepted target API request with required header.")
|
||||||
fingerprint_future.set_result(headers[FINGERPRINT_HEADER])
|
fingerprint_future.set_result(headers[FINGERPRINT_HEADER])
|
||||||
|
|
||||||
page.on("request", handle_request)
|
page.on("request", handle_request)
|
||||||
|
|
||||||
# [修改] 并发执行 goto 和 wait_for。拿到结果就立刻返回,不再死等 goto 结束
|
# [修改] 并发执行 goto 和 wait_for。拿到结果就立刻返回,不再死等 goto 结束
|
||||||
|
logger.info(f"Navigating to target page: {TARGET_PAGE}")
|
||||||
goto_task = asyncio.create_task(page.goto(TARGET_PAGE, wait_until="domcontentloaded"))
|
goto_task = asyncio.create_task(page.goto(TARGET_PAGE, wait_until="domcontentloaded"))
|
||||||
|
|
||||||
|
logger.info("Waiting for header generation...")
|
||||||
fingerprint = await asyncio.wait_for(fingerprint_future, timeout=15)
|
fingerprint = await asyncio.wait_for(fingerprint_future, timeout=15)
|
||||||
|
logger.info("Header successfully generated and retrieved.")
|
||||||
return fingerprint
|
return fingerprint
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
|
logger.error("Timeout while waiting for header generation.")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Unexpected error during generation: {str(e)}")
|
||||||
return None
|
return None
|
||||||
finally:
|
finally:
|
||||||
|
logger.info("Closing browser page...")
|
||||||
await page.close()
|
await page.close()
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
async def root():
|
async def root():
|
||||||
return {"status": "ok", "message": "广东移动指纹获取服务"}
|
return {"status": "ok", "message": "Web Header Signature Service is running"}
|
||||||
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
|
|
@ -116,9 +142,11 @@ async def health():
|
||||||
|
|
||||||
@app.get("/fingerprint")
|
@app.get("/fingerprint")
|
||||||
async def get_fingerprint_endpoint():
|
async def get_fingerprint_endpoint():
|
||||||
|
logger.info("Received request for new header signature")
|
||||||
try:
|
try:
|
||||||
fingerprint = await get_fingerprint()
|
fingerprint = await get_fingerprint()
|
||||||
if fingerprint:
|
if fingerprint:
|
||||||
|
logger.info("Successfully handled signature request")
|
||||||
return {
|
return {
|
||||||
"status": "success",
|
"status": "success",
|
||||||
"data": {
|
"data": {
|
||||||
|
|
@ -126,9 +154,11 @@ async def get_fingerprint_endpoint():
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
raise HTTPException(status_code=500, detail="获取指纹失败")
|
logger.error("Failed to generate signature (returned None)")
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to generate signature")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(status_code=500, detail=f"服务错误: {str(e)}")
|
logger.error(f"Service error during request: {str(e)}")
|
||||||
|
raise HTTPException(status_code=500, detail=f"Service error: {str(e)}")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue