@Universe Tech
Periodical Operations
IoT Integrations
Application Integrations
Automatic Tests for Web App
Web Crawlers
curl 'https://kktix.com/g/events/wdsegy-03/base_info' \
-H 'accept: */*' \
-H 'accept-language: zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6' \
-H 'cache-control: no-cache' \
-H 'cookie: locale=zh-TW; kktix_session_token_v2=b50b9089129186074beb3cb4ad19a94c; XSRF-TOKEN=EwZ9Gsxx2Ct86sQlCNhTjl39OGcGmYgubq0%2FNJj77jyRRWQD%2BvWJz3bUKfIp%2BOH6wzl6A4TBQqfFD29y9YirXw%3D%3D' \
-H 'pragma: no-cache' \
-H 'referer: https://kktix.com/events/wdsegy-03/registrations/new' \
-H 'sec-ch-ua: "Google Chrome";v="123", "Not:A-Brand";v="8", "Chromium";v="123"' \
-H 'sec-ch-ua-mobile: ?0' \
-H 'sec-ch-ua-platform: "macOS"' \
-H 'sec-fetch-dest: empty' \
-H 'sec-fetch-mode: cors' \
-H 'sec-fetch-site: same-origin' \
-H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36' \
-H 'x-csrf-token: EwZ9Gsxx2Ct86sQlCNhTjl39OGcGmYgubq0/NJj77jyRRWQD+vWJz3bUKfIp+OH6wzl6A4TBQqfFD29y9YirXw==' \
-H 'x-requested-with: XMLHttpRequest'
:status: 200
content-type: application/json
content-length: 2734
date: Sun, 31 Mar 2024 07:42:27 GMT
server: nginx
cache-control: no-cache
content-encoding: gzip
set-cookie: XSRF-TOKEN=ajQUXVXkqfPSFo2OfPhtoDl7%2Bw4fIoibBlFQI31%2BPvvodw1EY2D4F9goYFld2N%2FUp7%2B5ap16QhKt8wBlEA17mA%3D%3D; Path=/; Secure
vary: Accept-Encoding
x-cache: Miss from cloudfront
via: 1.1 a8e6d13425093fadbdcc8b1d50527eec.cloudfront.net (CloudFront)
x-amz-cf-pop: TPE51-C1
x-amz-cf-id: 4gSoR_FOgPBXSi7M-0ECgxqoM6Y_nKWWWL6PmSt495hpJObK99sisQ==
{"eventData":{"organization": {"contact_url":"http://kktix.com/organizations/chenhong/contact/new","fans_management_enabled":false,"google_analytics4_code":"","name":"辰紘行銷有限公司","public_domain":"chenhong.kktix.cc","show_contact_link":true,"slug":"chenhong"},
"event":{"accept_atm":true,"accept_creditcard":true,"accept_famiport":false,"accept_famiport_pay_and_pickup":false,"accept_paypal":false,"adult_only":false,"allow_not_continuous_seats":true,"arena":{"id":2898,"img":"https://t.kfs.io/upload_images/arena_images/2898/__02.jpg","name":"臺北市兒童新樂園","sections":{"45570":{"id":45570,"label":"0524全場域","needAllocation":false,"coords":[112,133,113,353,257,353,257,317,324,317,324,354,467,354,468,134,468,133,468,133],"flag":[464,145],"tooltip":[484,174],"pegman":[157,152],"extraText":"","seatStyle":"auto_seating","color":"#333333","situations":[],"areas":{"63841":{"applicable":[702259,702274,702275],"id":63841}},"position":4,"displaySectionName":true,"displayLineNo":false,"displaySeatNo":true},"45571":{"id":45571,"label":"0525全場域","needAllocation":false,"coords":[111,374,112,495,469,495,467,375,467,375],"flag":[443,514],"tooltip":[478,546],"pegman":[158,523],"extraText":"","seatStyle":"auto_seating","color":"#333333","situations":[],"areas":{"63842":{"applicable":[702260,702274,702275],"id":63842}},"position":2,"displaySectionName":true,"displayLineNo":false,"displaySeatNo":true}}},"artist_id":[],"benefits":"","booking_skippable":false,"booking_ttl":600,"booth_enabled":false,"cancelable_by_user":false,"capacity":0,"captcha_type":1,"co_organizer":"","culture_subsidy":null,"end_at":"2024-05-25T13:30:00Z","entry_info":"","event_currency":"TWD","famiport_enabled":true,"famiport_pay_and_pickup_enabled":false,"fun_coupon_type":"N","gcal_url":"https://www.google.com/calendar/event?action=TEMPLATE\u0026text=%E3%80%90%E9%A0%90%E5%94%AE%E7%A5%A8%E3%80%91%E5%8C%97%E6%8D%B730+%E6%98%A5%E5%AD%A3%E9%9F%B3%E6%A8%82PARTY\u0026dates=20240524T110000Z/20240525T133000Z\u0026details=https://chenhong.kktix.cc/events/wdsegy-03\u0026location=%E5%8F%B0%E5%8C%97%E5%B8%82%E5%A3%AB%E6%9E%97%E5%8D%80%E6%89%BF%E5%BE%B7%E8%B7%AF%E4%BA%94%E6%AE%B555%E8%99%9F\u0026trp=true\u0026sprop=https://chenhong.kktix.cc/events/wdsegy-03\u0026sprop=name:KKTIX","geo_lat":25.0981175,"geo_long":121.515366,"has_geo":true,"ibon_center_note_1":"主辦:台北捷運公司、辰紘行銷","ibon_center_note_2":"一人一票、憑票入場,預計18:30開放進場","ibon_center_note_3":"禁帶外食及攝(錄、相)機等入場,如有未盡事宜請詳售票頁面說明","ibon_enabled":false,"ibon_left_note_4":"本節目採級距式退票,請詳閱該節目頁退換票規定,退票期限以退票寄達日為準","id":102569,"invite_user_to_join_org_fan":false,"is_kktix_handle_refund":true,"is_published":true,"kktix_refund_due_before":null,"kktix_refund_fee_percentage":null,"kktix_refund_time_limit_type":"normal_d","location":"臺北市兒童新樂園 ","location_address":"台北市士林區承德路五段55號","mailing_enabled":false,"max_to_buy":4,"max_to_buy_per_user":4,"milli_kkpoint_magnification":null,"more_info":"","name":"【預售票】北捷30 春季音樂PARTY","need_booking":true,"need_mobile_verified":true,"og_image_url":"https://t.kfs.io/upload_images/194725/KKTIX1200x630_v03.jpg","order_data_deadline":null,"org_id":36501,"organizer":"","pickup_types":[{"enable":true,"expires_at":"2024-05-25T13:30:00Z","fee_cent":0,"type":"qrcode"},{"enable":true,"expires_at":"2024-06-01T15:59:59Z","fee_cent":3000,"type":"famiport"}],"plan_company":"WALKIE","policy":"","public_url":"https://chenhong.kktix.cc/events/wdsegy-03","qrcode_enabled":true,"qrcode_expires_at":"2024-05-25T13:30:00Z","register_ttl":600,"registration_theme":"vertical","sales_launch_count_down":true,"short_description":"北捷30 春季音樂PARTY","show_people":false,"show_registrant_serial":false,"slug":"wdsegy-03","start_at":"2024-05-24T11:00:00Z","support_quick_registration":false,"time_zone":"Asia/Taipei","type":null,"visible_level":0},
"payment_gateways":{"id":12701,"owner_type":"Organization","fee_twd_atm_payment_gateway":"ESUN_BANK_ATM_WALKIE_HOURLY","fee_twd_famiport_payment_gateway":"FAMIPORT_WALKIE","fee_twd_famiport_pay_and_pickup_payment_gateway":"","fee_twd_creditcard_payment_gateway":"NEWEBPAY_3D","fee_twd_paypal_payment_gateway":"","fee_hkd_creditcard_payment_gateway":"","fee_hkd_paypal_payment_gateway":"","fee_jpy_creditcard_payment_gateway":"","fee_jpy_paypal_payment_gateway":"","fee_usd_creditcard_payment_gateway":"","fee_usd_paypal_payment_gateway":"","fee_sgd_creditcard_payment_gateway":"","pickup_twd_booth":8000,"pickup_twd_mailing":8000,"pickup_hkd_mailing":500,"pickup_hkd_booth":1000,"company":"WALKIE","booking_fee":false},
"order_qualifications":[],
"tickets":[{"accounting_subject_id":null,"booking_fee_cents":0,"capacity":0,"disability_constraint":0,"end_at":"2024-05-24T10:00:00Z","end_at_for_registration":"2024-05-24T10:00:00Z","for_online_event":false,"id":702259,"limit_per_kkbox_uid":-1,"max_to_buy":4,"milli_kkpoint_magnification":null,"min_to_buy":1,"name":"預售單日票 05/24","need_invitation_code":false,"position":0,"price":{"cents":105000,"currency":"TWD"},"start_at":"2024-04-14T04:00:00Z","tax_free":false,"unit_to_buy":1,"unlimited_capacity":true},{"accounting_subject_id":null,"booking_fee_cents":0,"capacity":0,"disability_constraint":0,"end_at":"2024-05-25T10:00:00Z","end_at_for_registration":"2024-05-25T10:00:00Z","for_online_event":false,"id":702260,"limit_per_kkbox_uid":-1,"max_to_buy":4,"milli_kkpoint_magnification":null,"min_to_buy":1,"name":"預售單日票 05/25","need_invitation_code":false,"position":1,"price":{"cents":105000,"currency":"TWD"},"start_at":"2024-04-14T04:00:00Z","tax_free":false,"unit_to_buy":1,"unlimited_capacity":true}],
"stop_selling_tickets":[],
"contact_fields":[{"default_field":"name","description":null,"field_key":"field_text_876703","field_type":"text","i18n":{"name":{"default":"姓名","en":"Name","ja":"名前","zh-TW":"姓名"}},"id":876703,"is_required":true,"name":"姓名","options":null,"position":0},{"default_field":"email","description":null,"field_key":"field_email_876704","field_type":"email","i18n":{"name":{"default":"Email","en":"Email","ja":"Email","zh-TW":"Email"}},"id":876704,"is_required":true,"name":"Email","options":null,"position":1},{"default_field":"mobile","description":null,"field_key":"field_text_876705","field_type":"text","i18n":{"name":{"default":"手機","en":"Mobile","ja":"携帯番号","zh-TW":"手機"}},"id":876705,"is_required":true,"name":"手機","options":null,"position":2}],
"attendee_fields":[]}}
Login
Cookie-Session Based
Short-Term Token
Long-Term Token
Oauth2
Renew Token
Get first token manually?
Bypassing Anti-Bot Protection
CSRF Token
Application Defined Tokens
Login Example
Init first cookies
Request login with cookies
Save logined cookies
Send other requests with cookies
Login Example
protected function setCookiesAndToken()
{
$this->info('Resetting cookies and auth token...');
app('cookieJar')->clear();
$response = $this->client
->get(static::LOGIN_URI, ['cookies' => app('cookieJar')])
->getBody()
->getContents();
$pattern = '/<input(.*?)name=\"authenticity_token\"(.*)value=\"(.*?)\"/i';
preg_match($pattern, $response, $matches);
$this->authToken = end($matches);
return $this;
}
Login Example
protected function setLoginCookie()
{
$this->info('Log in with username: ' . $this->username . '...');
$loginResponse = $this->client->post(static::LOGIN_URI, [
'form_params' => [
'authenticity_token' => $this->authToken,
'user' => [
'login' => $this->username,
'password' => $this->password,
'remember_me' => '0',
]
],
'cookies' => app('cookieJar')
]);
$token = app('cookieJar')->getCookieByName('user_display_name_v2');
// ......
return $this;
}
Login Example
protected function setRegisterInfo()
{
$this->info("Checking status by event id: {$this->eventId}");
try {
$response = $this->client
->get(static::EVENT_URI . "{$this->eventId}/register_info", [
'cookies' => app('cookieJar')
])
->getBody()
->getContents();
} catch (ClientException $e) {
$this->error("Invalid event id: {$this->eventId}");
exit;
}
$result = json_decode($response, true);
$status = $result['inventory']['registerStatus'];
$this->ticketStatus = $result['inventory']['ticketInventory'];
$this->captchaContent = $result['ktx_captcha'] ?? null;
}
Request Headers
Referer
More than necessary cookies
More than necessary headers
Avoid using VPN/Proxy IPs
Man in the middle
Sniffing Tools
Fiddler
Charles
HTTP Toolkit
mitmproxy
Wiresharks
Captcha
Invisible Captcha
Detect Agents, like Headless Chrome, etc.
IP Detecting
TLS Handshake Detecting
Rate Limit for IP / Regions
Payload Encryption / Signature
SSL Pinging in Android / iOS
Captcha
OCR with Trained AI Model
ex: https://github.com/linsamtw/TaiwanTrainVerificationCode2text
Voice Recognition by AI
Selenium with Image Recognition
ex: https://blog.csdn.net/qq_52525445/article/details/122757053
Anti Captcha
https://anti-captcha.com
Anti Captcha
Native IP Proxy
https://www.lumiproxy.com/
https://www.lunaproxy.com/
https://www.iproyal.net/
https://www.webshare.io/residential-proxy
TLS Handshake
TLS Handshake
<body>
<div id="cf-wrapper">
<div class="cf-alert cf-alert-error cf-cookie-error" id="cookie-alert" data-translate="enable_cookies">Please enable cookies.</div>
<div id="cf-error-details" class="cf-error-details-wrapper">
<div class="cf-wrapper cf-header cf-error-overview">
<h1 data-translate="block_headline">Sorry, you have been blocked</h1>
<h2 class="cf-subheadline"><span data-translate="unable_to_access">You are unable to access</span> inline.app</h2>
</div><!-- /.header -->
<div class="cf-section cf-highlight">
<div class="cf-wrapper">
<div class="cf-screenshot-container cf-screenshot-full">
<span class="cf-no-screenshot error"></span>
</div>
</div>
</div><!-- /.captcha-container -->
<div class="cf-section cf-wrapper">
<div class="cf-columns two">
<div class="cf-column">
<h2 data-translate="blocked_why_headline">Why have I been blocked?</h2>
<p data-translate="blocked_why_detail">This website is using a security service to protect itself from online attacks. The action you just performed triggered the security solution. There are several actions that could trigger this block including submitting a certain word or phrase, a SQL command or malformed data.</p>
</div>
curl https://inline.app/api/menus?companyId=-LamXb5SAQN7JcJfyRKi%3Ainline-live-2a466&time=2024-03-31T00%3A00%3A00.000Z
curl-impersonate
A special build of curl that can impersonate the four major browsers: Chrome, Edge, Safari & Firefox. curl-impersonate is able to perform TLS and HTTP handshakes that are identical to that of a real browser.
curl-impersonate can be used either as a command line tool, similar to the regular curl, or as a library that can be integrated instead of the regular libcurl.
https://github.com/lwthiker/curl-impersonate
use HeadlessChromium\BrowserFactory;
$browserFactory = new BrowserFactory();
// starts headless Chrome
$browser = $browserFactory->createBrowser();
try {
// creates a new page and navigate to an URL
$page = $browser->createPage();
$page->navigate('http://example.com')->waitForNavigation();
// get page title
$pageTitle = $page->evaluate('document.title')->getReturnValue();
// screenshot - Say "Cheese"! 😄
$page->screenshot()->saveToFile('/foo/bar.png');
// pdf
$page->pdf(['printBackground' => false])->saveToFile('/foo/bar.pdf');
} finally {
// bye
$browser->close();
}
Scrapfly
https://scrapfly.io/