需求场景

最近在写Golang爬虫的时候，需要获取请求的Cookie、Header等信息，每次都从F12里面获取，很不优雅，也很不方便，就想着能不能从curl中解析，然后就发现了这个库，在此记录一下。

介绍

pcurl是解析curl命令的库，弥补go生态链的一块空白[从零实现]

仓库地址：https://github.com/antlabs/pcurl

feature

  • 支持-X; --request，作用设置GET或POST的选项
  • 支持-H; --header选项，curl中用于设置http header的选项
  • 支持-d; --data选项，作用设置http body
  • 支持--data-raw选项，curl用于设置http body
  • 支持-F --form选项，用作设置formdata
  • 支持--url选项，curl中设置url，一般不会设置这个选项
  • 支持--compressed选项
  • 支持-k, --insecure选项
  • 支持-G, --get选项
  • 支持--data-urlencode选项
  • 支持内嵌到你的结构体里面，让你的cmd秒变curl

快速开始

这里以博客中获取统计信息的接口为例：

以cURL格式复制：

curl 'https://blog.kakkk.net/?action=get_statistic&size=2048' \
  -H 'authority: blog.kakkk.net' \
  -H 'accept: application/json, text/javascript, */*; q=0.01' \
  -H 'accept-language: zh-CN,zh;q=0.9' \
  -H 'cookie: theme_dark=1' \
  -H 'referer: https://blog.kakkk.net/' \
  -H 'sec-ch-ua: "Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"' \
  -H 'sec-ch-ua-mobile: ?0' \
  -H 'sec-ch-ua-platform: "macOS"' \
  -H 'sec-fetch-dest: empty' \
  -H 'sec-fetch-mode: cors' \
  -H 'sec-fetch-site: same-origin' \
  -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' \
  -H 'x-requested-with: XMLHttpRequest' \
  --compressed

golang测试：

package main

import (
    "fmt"
    "github.com/antlabs/pcurl"
    "io"
    "net/http"
    "os"
)

func main() {
    req, err := pcurl.ParseAndRequest(`curl 'https://blog.kakkk.net/?action=get_statistic&size=2048' \
  -H 'authority: blog.kakkk.net' \
  -H 'accept: application/json, text/javascript, */*; q=0.01' \
  -H 'accept-language: zh-CN,zh;q=0.9' \
  -H 'cookie: theme_dark=1' \
  -H 'referer: https://blog.kakkk.net/' \
  -H 'sec-ch-ua: "Chromium";v="104", " Not A;Brand";v="99", "Google Chrome";v="104"' \
  -H 'sec-ch-ua-mobile: ?0' \
  -H 'sec-ch-ua-platform: "macOS"' \
  -H 'sec-fetch-dest: empty' \
  -H 'sec-fetch-mode: cors' \
  -H 'sec-fetch-site: same-origin' \
  -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36' \
  -H 'x-requested-with: XMLHttpRequest' \
  --compressed`)
    if err != nil {
        fmt.Printf("err:%s\n", err)
        return
    }
    fmt.Printf("authority: %v\n", req.Header.Get("authority"))
    fmt.Printf("cookie: %v\n", req.Header.Get("cookie"))
    fmt.Printf("user-agent: %v\n", req.Header.Get("user-agent"))
    resp, err := http.DefaultClient.Do(req)
    if err != nil {
        fmt.Printf("err:%s\n", err)
        return
    }
    defer resp.Body.Close()
    io.Copy(os.Stdout, resp.Body)
}

运行结果：

authority:  blog.kakkk.net
cookie:  theme_dark=1
user-agent:  Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36
{"post_calendar":{"series":[["2021-09-25",1],["2021-09-26",1],["2021-09-27",1],["2021-09-28",0],["2021-09-29",1],["2021-09-30",0],["2021-10-01",0],["2021-10-02",0],["2021-10-03",0],["2021-10-04",0],["2021-10-05",0],["2021-10-06",0],["2021-10-07",0],["2021-10-08",0],["2021-10-09",0],["2021-10-10",0],["2021-10-11",0],["2021-10-12",0],["2021-10-13",0],["2021-10-14",0],["2021-10-15",0],["2021-10-16",0],["2021-10-17",0],["2021-10-18",0],["2021-10-19",0],["2021-10-20",0],["2021-10-21",0],["2021-10-22",0],["2021-10-23",1],["2021-10-24",0],["2021-10-25",0],["2021-10-26",0],["2021-10-27",0],["2021-10-28",0],["2021-10-29",0],["2021-10-30",0],["2021-10-31",0],["2021-11-01",0],["2021-11-02",0],["2021-11-03",0],["2021-11-04",0],["2021-11-05",0],["2021-11-06",0],["2021-11-07",0],["2021-11-08",0],["2021-11-09",1],["2021-11-10",0],["2021-11-11",0],["2021-11-12",0],["2021-11-13",0],["2021-11-14",0],["2021-11-15",0],["2021-11-16",0],["2021-11-17",1],["2021-11-18",1],["2021-11-19",0],["2021-11-20",0],["2021-11-21",0],["2021-11-22",0],["2021-11-23",2],["2021-11-24",0],["2021-11-25",0],["2021-11-26",1],["2021-11-27",1],["2021-11-28",0],["2021-11-29",0],["2021-11-30",0],["2021-12-01",0],["2021-12-02",0],["2021-12-03",1],["2021-12-04",0],["2021-12-05",0],["2021-12-06",0],["2021-12-07",0],["2021-12-08",1],["2021-12-09",0],["2021-12-10",0],["2021-12-11",1],["2021-12-12",0],["2021-12-13",0],["2021-12-14",0],["2021-12-15",0],["2021-12-16",1],["2021-12-17",1],["2021-12-18",1],["2021-12-19",0],["2021-12-20",0],["2021-12-21",0],["2021-12-22",0],["2021-12-23",0],["2021-12-24",0],["2021-12-25",0],["2021-12-26",0],["2021-12-27",1],["2021-12-28",2],["2021-12-29",1],["2021-12-30",2],["2021-12-31",1],["2022-01-01",0],["2022-01-02",0],["2022-01-03",0],["2022-01-04",0],["2022-01-05",0],["2022-01-06",0],["2022-01-07",1],["2022-01-08",1],["2022-01-09",0],["2022-01-10",0],["2022-01-11",1],["2022-01-12",0],["2022-01-13",0],["2022-01-14",0],["2022-01-15",0],["2022-01-16",3],["2022-01-17",0],["2022-01-18",0],["2022-01-19",0],["2022-01-20",0],["2022-01-21",0],["2022-01-22",0],["2022-01-23",0],["2022-01-24",0],["2022-01-25",0],["2022-01-26",0],["2022-01-27",0],["2022-01-28",0],["2022-01-29",0],["2022-01-30",0],["2022-01-31",0],["2022-02-01",0],["2022-02-02",0],["2022-02-03",0],["2022-02-04",0],["2022-02-05",0],["2022-02-06",0],["2022-02-07",0],["2022-02-08",0],["2022-02-09",0],["2022-02-10",0],["2022-02-11",0],["2022-02-12",0],["2022-02-13",0],["2022-02-14",0],["2022-02-15",0],["2022-02-16",0],["2022-02-17",0],["2022-02-18",0],["2022-02-19",0],["2022-02-20",0],["2022-02-21",0],["2022-02-22",0],["2022-02-23",0],["2022-02-24",0],["2022-02-25",0],["2022-02-26",0],["2022-02-27",0],["2022-02-28",0],["2022-03-01",0],["2022-03-02",0],["2022-03-03",0],["2022-03-04",0],["2022-03-05",0],["2022-03-06",0],["2022-03-07",0],["2022-03-08",1],["2022-03-09",0],["2022-03-10",0],["2022-03-11",0],["2022-03-12",0],["2022-03-13",0],["2022-03-14",0],["2022-03-15",0],["2022-03-16",0],["2022-03-17",0],["2022-03-18",0],["2022-03-19",0],["2022-03-20",2],["2022-03-21",0],["2022-03-22",0],["2022-03-23",0],["2022-03-24",0],["2022-03-25",0],["2022-03-26",0],["2022-03-27",0],["2022-03-28",0],["2022-03-29",0],["2022-03-30",0],["2022-03-31",0],["2022-04-01",0],["2022-04-02",0],["2022-04-03",0],["2022-04-04",0],["2022-04-05",0],["2022-04-06",0],["2022-04-07",0],["2022-04-08",0],["2022-04-09",0],["2022-04-10",0],["2022-04-11",0],["2022-04-12",0],["2022-04-13",0],["2022-04-14",0],["2022-04-15",0],["2022-04-16",1],["2022-04-17",2],["2022-04-18",0],["2022-04-19",0],["2022-04-20",0],["2022-04-21",0],["2022-04-22",0],["2022-04-23",1],["2022-04-24",0],["2022-04-25",1],["2022-04-26",0],["2022-04-27",0],["2022-04-28",0],["2022-04-29",0],["2022-04-30",0],["2022-05-01",0],["2022-05-02",0],["2022-05-03",0],["2022-05-04",0],["2022-05-05",0],["2022-05-06",0],["2022-05-07",0],["2022-05-08",0],["2022-05-09",0],["2022-05-10",0],["2022-05-11",1],["2022-05-12",1],["2022-05-13",0],["2022-05-14",0],["2022-05-15",0],["2022-05-16",1],["2022-05-17",0],["2022-05-18",0],["2022-05-19",0],["2022-05-20",0],["2022-05-21",0],["2022-05-22",0],["2022-05-23",0],["2022-05-24",0],["2022-05-25",0],["2022-05-26",0],["2022-05-27",0],["2022-05-28",0],["2022-05-29",0],["2022-05-30",0],["2022-05-31",0],["2022-06-01",0],["2022-06-02",0],["2022-06-03",0],["2022-06-04",1],["2022-06-05",0],["2022-06-06",0],["2022-06-07",0],["2022-06-08",0],["2022-06-09",0],["2022-06-10",0],["2022-06-11",0],["2022-06-12",0],["2022-06-13",0],["2022-06-14",0],["2022-06-15",0],["2022-06-16",0],["2022-06-17",0],["2022-06-18",0],["2022-06-19",0],["2022-06-20",0],["2022-06-21",0],["2022-06-22",0],["2022-06-23",0],["2022-06-24",0],["2022-06-25",0],["2022-06-26",0],["2022-06-27",0],["2022-06-28",0],["2022-06-29",1],["2022-06-30",0],["2022-07-01",0],["2022-07-02",1],["2022-07-03",1],["2022-07-04",2],["2022-07-05",0],["2022-07-06",0],["2022-07-07",0],["2022-07-08",0],["2022-07-09",0],["2022-07-10",0],["2022-07-11",0],["2022-07-12",0],["2022-07-13",0],["2022-07-14",0],["2022-07-15",0],["2022-07-16",1],["2022-07-17",2],["2022-07-18",0],["2022-07-19",0],["2022-07-20",0],["2022-07-21",0],["2022-07-22",0],["2022-07-23",0],["2022-07-24",0],["2022-07-25",0],["2022-07-26",0],["2022-07-27",0],["2022-07-28",0],["2022-07-29",0],["2022-07-30",0],["2022-07-31",0],["2022-08-01",0],["2022-08-02",0],["2022-08-03",0],["2022-08-04",0],["2022-08-05",0],["2022-08-06",0],["2022-08-07",0],["2022-08-08",0],["2022-08-09",0],["2022-08-10",0],["2022-08-11",0],["2022-08-12",0],["2022-08-13",0],["2022-08-14",0],["2022-08-15",0],["2022-08-16",0],["2022-08-17",0],["2022-08-18",0],["2022-08-19",0],["2022-08-20",0],["2022-08-21",0],["2022-08-22",0],["2022-08-23",0],["2022-08-24",0],["2022-08-25",0]],"range":["2021-9-25","2022-8-25"],"categories":[0,1,2,3],"max":3},"post_chart":{"series":[3,4,3,2,6,0],"xAxis":["2022-03","2022-04","2022-05","2022-06","2022-07","2022-08"],"categories":[0,1,2,3,4,5,6]},"category_radar":{"indicator":[{"name":"Golang","max":"27"},{"name":"Java","max":"27"},{"name":"\u6570\u636e\u5e93","max":"27"},{"name":"C#","max":"27"},{"name":"\u6570\u636e\u7ed3\u6784\u4e0e\u7b97\u6cd5","max":"27"},{"name":"\u524d\u7aef","max":"27"},{"name":"Linux","max":"27"},{"name":"\u5b66\u4e60\u7b14\u8bb0","max":"27"},{"name":"Leetcode","max":"27"},{"name":"\u5176\u4ed6","max":"27"}],"series":["27","17","13","9","21","5","6","8","6","14"]},"categories_chart":{"color":["#6772e5","#ff9e0f","#fa755a","#3ecf8e","#82d3f4","#ab47bc","#525f7f","#f51c47","#26A69A","#6772e5"],"indicator":[{"name":"Golang","value":"27"},{"name":"Java","value":"17"},{"name":"\u6570\u636e\u5e93","value":"13"},{"name":"C#","value":"9"},{"name":"\u6570\u636e\u7ed3\u6784\u4e0e\u7b97\u6cd5","value":"21"},{"name":"\u524d\u7aef","value":"5"},{"name":"Linux","value":"6"},{"name":"\u5b66\u4e60\u7b14\u8bb0","value":"8"},{"name":"Leetcode","value":"6"},{"name":"\u5176\u4ed6","value":"14"}],"series":["27","17","13","9","21","5","6","8","6","14"]},"tags_chart":{"indicator":["Golang","\u6570\u636e\u7ed3\u6784","\u6570\u636e\u5e93","\u5b66\u4e60\u7b14\u8bb0","MySQL","C#",".NET","\u8e29\u8fc7\u7684\u5751"],"series":["24","12","11","10","10","9","9","7"]}}

其他用法看仓库介绍

最后修改：2022 年 08 月 25 日
© 允许规范转载
如果觉得我的文章对你有用，请随意赞赏