From 5e9850414bfa7965f44d9fd5301a596061110a73 Mon Sep 17 00:00:00 2001 From: Marcus Rohrmoser Date: Sun, 6 Jan 2019 22:43:50 +0100 Subject: [PATCH] basic Go pinboard API. https://github.com/mro/Shaarli-API-test/issues/7 --- pinboard.go | 279 +++++++++++++++++++++++++++++++++++++++ pinboard.sh | 64 +++++++++ pinboard_test.go | 151 +++++++++++++++++++++ testdata/bookmark/login.html | 64 +++++++++ testdata/sebsauvage/login.html | 51 +++++++ testdata/shaarligo/linkform.html | 25 ++++ testdata/v0.10.2/login.html | 201 ++++++++++++++++++++++++++++ version.go | 3 + 8 files changed, 838 insertions(+) create mode 100644 pinboard.go create mode 100755 pinboard.sh create mode 100644 pinboard_test.go create mode 100644 testdata/bookmark/login.html create mode 100644 testdata/sebsauvage/login.html create mode 100644 testdata/shaarligo/linkform.html create mode 100644 testdata/v0.10.2/login.html create mode 100644 version.go diff --git a/pinboard.go b/pinboard.go new file mode 100644 index 0000000..9fc58e1 --- /dev/null +++ b/pinboard.go @@ -0,0 +1,279 @@ +// +// Copyright (C) 2019-2019 Marcus Rohrmoser, https://code.mro.name/mro/Shaarli-API-test +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +package main + +import ( + "github.com/yhat/scrape" + "golang.org/x/net/html" + "golang.org/x/net/html/atom" + "golang.org/x/net/publicsuffix" + "io" + "log" + "net/http" + "net/http/cgi" + "net/http/cookiejar" + "net/url" + "os" + "strings" + "time" +) + +var GitSHA1 = "Please set -ldflags \"-X main.GitSHA1=$(git rev-parse --short HEAD)\"" // https://medium.com/@joshroppo/setting-go-1-5-variables-at-compile-time-for-versioning-5b30a965d33e + +// even cooler: https://stackoverflow.com/a/8363629 +// +// inspired by // https://coderwall.com/p/cp5fya/measuring-execution-time-in-go +func trace(name string) (string, time.Time) { return name, time.Now() } +func un(name string, start time.Time) { log.Printf("%s took %s", name, time.Since(start)) } + +func main() { + if true { + // lighttpd doesn't seem to like more than one (per-vhost) server.breakagelog + log.SetOutput(os.Stderr) + } else { // log to custom logfile rather than stderr (may not be reachable on shared hosting) + } + + if err := cgi.Serve(http.HandlerFunc(handleMux)); err != nil { + log.Fatal(err) + } +} + +// https://pinboard.in/api +// +// All API methods are GET requests, even when good REST habits suggest they should use a different verb. +// +// v1/posts/add +// v1/posts/delete +// v1/posts/get +func handleMux(w http.ResponseWriter, r *http.Request) { + defer un(trace(strings.Join([]string{"v", version, "+", GitSHA1, " ", r.RemoteAddr, " ", r.Method, " ", r.URL.String()}, ""))) + // w.Header().Set("Server", strings.Join([]string{myselfNamespace, CurrentShaarliGoVersion}, "#")) + // w.Header().Set("X-Powered-By", strings.Join([]string{myselfNamespace, CurrentShaarliGoVersion}, "#")) + // now := time.Now() + + path_info := os.Getenv("PATH_INFO") + base := *r.URL + base.Path = base.Path[0:len(base.Path)-len(path_info)] + "/../index.php" + // script_name := os.Getenv("SCRIPT_NAME") + // urlBase := mustParseURL(string(xmlBaseFromRequestURL(r.URL, os.Getenv("SCRIPT_NAME")))) + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + switch path_info { + case "/v1/info": + io.WriteString(w, "r.URL: "+r.URL.String()+"\n") + io.WriteString(w, "base: "+base.String()+"\n") + + return + case "/v1/posts/add": + uid, pwd, ok := r.BasicAuth() + if !ok { + http.Error(w, "Basic Pre-Authentication required.", http.StatusUnauthorized) + return + } + + if "GET" != r.Method { + http.Error(w, "All API methods are GET requests, even when good REST habits suggest they should use a different verb.", http.StatusMethodNotAllowed) + return + } + + params := r.URL.Query() + if 1 != len(params["url"]) { + http.Error(w, "Required parameter missing: url", http.StatusBadRequest) + return + } + p_url := params["url"][0] + + if 1 != len(params["description"]) { + http.Error(w, "Required parameter missing: description", http.StatusBadRequest) + return + } + p_description := params["description"][0] + + p_extended := "" + if 1 == len(params["extended"]) { + p_extended = params["extended"][0] + } + + p_tags := "" + if 1 == len(params["tags"]) { + p_tags = params["tags"][0] + } + + v := url.Values{} + v.Set("post", p_url) + v.Set("title", p_description) + base.RawQuery = v.Encode() + + // https://stackoverflow.com/a/18414432 + options := cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + } + jar, err := cookiejar.New(&options) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + client := http.Client{Jar: jar} + resp, err := client.Get(base.String()) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + form, err := formValuesFromReader(resp.Body, "loginform") + resp.Body.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + form.Set("login", uid) + form.Set("password", pwd) + form.Set("returnurl", r.URL.String()) + resp, err = client.PostForm(resp.Request.URL.String(), form) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + form, err = formValuesFromReader(resp.Body, "linkform") + resp.Body.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + // if we do not have a linkform, auth must have failed. + if 0 == len(form) { + http.Error(w, "Authentication failed", http.StatusForbidden) + return + } + + // form.Set("lf_linkdate", "20190106_172531") + form.Set("lf_url", p_url) + form.Set("lf_title", p_description) + form.Set("lf_description", p_extended) + form.Set("lf_tags", p_tags) + + resp, err = client.PostForm(resp.Request.URL.String(), form) + resp.Body.Close() + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + + w.Header().Set("Content-Type", "text/xml; charset=utf-8") + io.WriteString(w, "") + return + case "/v1/posts/delete": + _, _, ok := r.BasicAuth() + if !ok { + http.Error(w, "Basic Pre-Authentication required.", http.StatusUnauthorized) + return + } + + if "GET" != r.Method { + http.Error(w, "All API methods are GET requests, even when good REST habits suggest they should use a different verb.", http.StatusMethodNotAllowed) + return + } + + params := r.URL.Query() + if 1 != len(params["url"]) { + http.Error(w, "Required parameter missing: url", http.StatusBadRequest) + return + } + // p_url := params["url"][0] + + io.WriteString(w, "bhb") + return + case "/v1/posts/update": + _, _, ok := r.BasicAuth() + if !ok { + http.Error(w, "Basic Pre-Authentication required.", http.StatusUnauthorized) + return + } + + if "GET" != r.Method { + http.Error(w, "All API methods are GET requests, even when good REST habits suggest they should use a different verb.", http.StatusMethodNotAllowed) + return + } + + w.Header().Set("Content-Type", "text/xml; charset=utf-8") + io.WriteString(w, "") + return + case "/v1/posts/get": + _, _, ok := r.BasicAuth() + if !ok { + http.Error(w, "Basic Pre-Authentication required.", http.StatusUnauthorized) + return + } + + if "GET" != r.Method { + http.Error(w, "All API methods are GET requests, even when good REST habits suggest they should use a different verb.", http.StatusMethodNotAllowed) + return + } + + params := r.URL.Query() + if 1 != len(params["url"]) { + http.Error(w, "Required parameter missing: url", http.StatusBadRequest) + return + } + // p_url := params["url"][0] + + return + case "/v1/posts/recent": + case "/v1/posts/dates": + case "/v1/posts/suggest": + case "/v1/tags/get": + case "/v1/tags/delete": + case "/v1/tags/rename": + case "/v1/user/secret": + case "/v1/user/api_token": + case "/v1/notes/list": + case "/v1/notes/ID": + http.Error(w, "Not Implemented", http.StatusNotImplemented) + return + } + http.NotFound(w, r) +} + +func formValuesFromReader(r io.Reader, name string) (ret url.Values, err error) { + root, err := html.Parse(r) + if err != nil { + return ret, err + } + + for _, form := range scrape.FindAll(root, func(n *html.Node) bool { return atom.Form == n.DataAtom }) { + if name != scrape.Attr(form, "name") && name != scrape.Attr(form, "id") { + continue + } + ret := url.Values{} + for _, inp := range scrape.FindAll(form, func(n *html.Node) bool { return atom.Input == n.DataAtom || atom.Textarea == n.DataAtom }) { + n := scrape.Attr(inp, "name") + if n == "" { + n = scrape.Attr(inp, "id") + } + ty := scrape.Attr(inp, "type") + v := scrape.Attr(inp, "value") + if v == "" && ty == "checkbox" { + v = scrape.Attr(inp, "checked") + } + ret.Set(n, v) + } + return ret, err // return on first occurence + } + return ret, err +} diff --git a/pinboard.sh b/pinboard.sh new file mode 100755 index 0000000..446e31d --- /dev/null +++ b/pinboard.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# https://golang.org/doc/install/source#environment +# + +cd "$(dirname "${0}")" || exit 1 +# $ uname -s -m +# Darwin x86_64 +# Linux x86_64 +# Linux armv6l + +say="say" +parm="" # "-u" +{ + "${say}" "go get" + go get ${parm} \ + golang.org/x/net/html \ + \ + github.com/stretchr/testify \ + github.com/yhat/scrape +} + +PROG_NAME="pinboard" +VERSION="$(grep -F 'version = ' version.go | cut -d \" -f 2)" + +rm "${PROG_NAME}"-*-"${VERSION}" 2>/dev/null + +"${say}" "test" +umask 0022 +go fmt && go vet && go test --short || { exit $?; } +"${say}" "ok" + +"${say}" "build localhost" +go build -ldflags "-s -w -X main.GitSHA1=$(git rev-parse --short HEAD)" -o ~/public_html/b/pinboard.cgi || { echo "Aua" 1>&2 && exit 1; } +"${say}" "ok" +# open "http://localhost/~$(whoami)/b/pinboard.cgi" + +"${say}" bench +go test -bench=. +"${say}" ok + +"${say}" "linux build" +# http://dave.cheney.net/2015/08/22/cross-compilation-with-go-1-5 +env GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.GitSHA1=$(git rev-parse --short HEAD)" -o "${PROG_NAME}-linux-amd64-${VERSION}" || { echo "Aua" 1>&2 && exit 1; } +# env GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-s -w -X main.GitSHA1=$(git rev-parse --short HEAD)" -o "${PROG_NAME}-linux-arm-${VERSION}" || { echo "Aua" 1>&2 && exit 1; } +# env GOOS=linux GOARCH=386 GO386=387 go build -o "${PROG_NAME}-linux-386-${VERSION}" # https://github.com/golang/go/issues/11631 +# env GOOS=darwin GOARCH=amd64 go build -o "${PROG_NAME}-darwin-amd64-${VERSION}" + + +"${say}" "simply" +# scp "ServerInfo.cgi" simply:/var/www/lighttpd/h4u.r-2.eu/public_html/"info.cgi" +gzip --force --best "${PROG_NAME}"-*-"${VERSION}" \ +&& chmod a-x "${PROG_NAME}"-*-"${VERSION}.gz" \ +&& rsync -vp --bwlimit=1234 "${PROG_NAME}"-*-"${VERSION}.gz" "simply:/tmp/" \ +&& ssh simply "sh -c 'cd /var/www/lighttpd/l.mro.name/public_html/ && cp "/tmp/${PROG_NAME}-linux-amd64-${VERSION}.gz" pinboard_cgi.gz && gunzip < pinboard_cgi.gz > pinboard.cgi && chmod a+x pinboard.cgi && ls -l pinboard?cgi*'" \ +&& ssh simply "sh -c 'cd /var/www/lighttpd/b.r-2.eu/public_html/u/ && cp /var/www/lighttpd/l.mro.name/public_html/pinboard?cgi* .'" + +ssh simply "sh -c 'cd /var/www/lighttpd/b.mro.name/public_html/u/ && cp /var/www/lighttpd/l.mro.name/public_html/pinboard?cgi* . && ls -l pinboard?cgi*'" +"${say}" "ok" + +"${say}" "vario" +# scp "ServerInfo.cgi" vario:~/mro.name/webroot/b/"info.cgi" +ssh vario "sh -c 'cd ~/mro.name/webroot/b/ && curl -L http://purl.mro.name/pinboard_cgi.gz | tee pinboard_cgi.gz | gunzip > pinboard.cgi && chmod a+x pinboard.cgi && ls -l pinboard?cgi*'" +"${say}" "ok" + diff --git a/pinboard_test.go b/pinboard_test.go new file mode 100644 index 0000000..32469a6 --- /dev/null +++ b/pinboard_test.go @@ -0,0 +1,151 @@ +// +// Copyright (C) 2019-2019 Marcus Rohrmoser, https://code.mro.name/mro/Shaarli-API-test +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// + +package main + +import ( + "fmt" + "net/url" + "os" + + "github.com/stretchr/testify/assert" + "testing" +) + +func TestURL(t *testing.T) { + t.Parallel() + + u, _ := url.Parse("https://l.mro.name/pinboard.cgi/v1/info") + assert.Equal(t, "https://l.mro.name/pinboard.cgi/v1/info", u.String(), "ach") + + base := *u + assert.Equal(t, "https://l.mro.name/pinboard.cgi/v1/info", base.String(), "ach") + + path_info := "/v1/info" + base.Path = base.Path[0:len(base.Path)-len(path_info)] + "/../index.php" + assert.Equal(t, "https://l.mro.name/pinboard.cgi/../index.php", base.String(), "ach") + + v := url.Values{} + v.Set("post", "uhu") + base.RawQuery = v.Encode() + assert.Equal(t, "https://l.mro.name/pinboard.cgi/../index.php?post=uhu", base.String(), "ach") +} + +func TestFormValuesFromHtml(t *testing.T) { + file, err := os.Open("testdata/v0.10.2/login.html") // curl --location --output testdata/login.html 'https://demo.shaarli.org/?post=https://demo.mro.name/shaarligo' + assert.Nil(t, err, "soso") + ips, _ := formValuesFromReader(file, "loginform") + assert.Equal(t, 40, len(ips["token"][0]), "form.token") + // assert.Equal(t, "", ips["returnurl"][0], "form.returnurl") + assert.Equal(t, "Login", ips[""][0], "form.") + assert.Equal(t, "", ips["login"][0], "form.login") + assert.Equal(t, "", ips["password"][0], "form.password") + assert.Equal(t, "", ips["longlastingsession"][0], "form.longlastingsession") + file.Close() + + file, err = os.Open("testdata/sebsauvage/login.html") // curl --location --output testdata/login.html 'https://demo.shaarli.org/?post=https://demo.mro.name/shaarligo' + assert.Nil(t, err, "soso") + ips, _ = formValuesFromReader(file, "loginform") + assert.Equal(t, 40, len(ips["token"][0]), "form.token") + // assert.Equal(t, "", ips["returnurl"][0], "form.returnurl") + assert.Equal(t, "Login", ips[""][0], "form.") + assert.Equal(t, "", ips["login"][0], "form.login") + assert.Equal(t, "", ips["password"][0], "form.password") + assert.Equal(t, "", ips["longlastingsession"][0], "form.longlastingsession") + file.Close() + + file, err = os.Open("testdata/bookmark/login.html") // curl --location --output testdata/login.html 'https://demo.shaarli.org/?post=https://demo.mro.name/shaarligo' + assert.Nil(t, err, "soso") + ips, _ = formValuesFromReader(file, "loginform") + assert.Equal(t, 40, len(ips["token"][0]), "form.token") + // assert.Equal(t, "", ips["returnurl"][0], "form.returnurl") + assert.Equal(t, "Login", ips[""][0], "form.") + assert.Equal(t, "", ips["login"][0], "form.login") + assert.Equal(t, "", ips["password"][0], "form.password") + assert.Equal(t, "", ips["longlastingsession"][0], "form.longlastingsession") + file.Close() + + file, err = os.Open("testdata/shaarligo/linkform.html") // curl --location --output testdata/login.html 'https://demo.shaarli.org/?post=https://demo.mro.name/shaarligo' + assert.Nil(t, err, "soso") + ips, _ = formValuesFromReader(file, "linkform") + assert.Equal(t, 40, len(ips["token"][0]), "form.token") + assert.Equal(t, "", ips["returnurl"][0], "form.returnurl") + assert.Equal(t, "20190106_172531", ips["lf_linkdate"][0], "form.lf_linkdate") + assert.Equal(t, "", ips["lf_url"][0], "form.lf_url") + assert.Equal(t, "uhu", ips["lf_title"][0], "form.lf_title") + assert.Equal(t, "", ips["lf_description"][0], "form.lf_description") + assert.Equal(t, "", ips["lf_tags"][0], "form.lf_tags") + assert.Equal(t, "Save", ips["save_edit"][0], "form.save_edit") + file.Close() + + /* + + + + + { 🔗 🐳 🚀 💫 } + + +
+ + + + + + + + + + + +
+ + + */ +} + +func TestRailRoad(t *testing.T) { + fa := func(success bool) (ret string, err error) { + if !success { + return "fail", fmt.Errorf("failure") + } + return "ok", err + } + fb := func(s string, err0 error) (int, error) { + if err0 != nil { + return 0, err0 + } + ret := 1 + if s == "ok" { + ret = 2 + } + return ret, err0 + } + { + v, e := fb(fa(true)) + assert.Equal(t, 2, v, "success") + assert.Equal(t, nil, e, "success") + } + { + v, e := fb(fa(false)) + assert.Equal(t, 0, v, "success") + assert.Equal(t, "failure", e.Error(), "success") + } +} diff --git a/testdata/bookmark/login.html b/testdata/bookmark/login.html new file mode 100644 index 0000000..8885453 --- /dev/null +++ b/testdata/bookmark/login.html @@ -0,0 +1,64 @@ + + +my bookmarks + + + + + + + + + + + + + + + + + + + + + + diff --git a/testdata/sebsauvage/login.html b/testdata/sebsauvage/login.html new file mode 100644 index 0000000..715fd97 --- /dev/null +++ b/testdata/sebsauvage/login.html @@ -0,0 +1,51 @@ + + +Liens en vrac de sebsauvage + + + + + + + + + + + + + + + diff --git a/testdata/shaarligo/linkform.html b/testdata/shaarligo/linkform.html new file mode 100644 index 0000000..1815e57 --- /dev/null +++ b/testdata/shaarligo/linkform.html @@ -0,0 +1,25 @@ + + + + +{ 🔗 🐳 🚀 💫 } + + +
+ + + + + + + + + + + +
+ + diff --git a/testdata/v0.10.2/login.html b/testdata/v0.10.2/login.html new file mode 100644 index 0000000..f197f72 --- /dev/null +++ b/testdata/v0.10.2/login.html @@ -0,0 +1,201 @@ + + + + Login - Shaarli demo (master) + + + + + + + + + + + + + + + + +
+
+
+ + + Shaarli demo (master) + + +
+
+
+
+ +
+ +
+
+
+
+ +
+ +
+
+
+ Delete +
+
+
+ +
+ +
+ + + + + + + +
+ + +
+
+ +
+
+ + +
+ +
+
+ +
+
+ + + + + + + + + + + + + + diff --git a/version.go b/version.go new file mode 100644 index 0000000..c3f82fa --- /dev/null +++ b/version.go @@ -0,0 +1,3 @@ +package main + +const version = "0.0.1" -- 2.11.4.GIT