pvl-tools/tab/1.cson - master - githubmirror/client

pvl-tools/tab/1.cson
364 строки16 KB
Michał Zochniak
Move reddit PVL to HTML (#29325)
25 июн 2026, 23:36
25 июн 2026, 23:365a264b8
pvl_version: 1
revision: 1
services:
  coinbase: [[
    # coinbase proofs are not supported anymore
    # all checks fail
    { fill: {
      , with: "x"
      , into: "tmp1" } },
    { assert_regex_match: {
      , pattern: "^y$"
      , from: "tmp1"
      , error: ["SERVICE_DEAD", "coinbase proofs are no longer supported"] } },
  ]]
  dns: [[
    # DNS has no hint. And it checks every txt record on two domains. And errors are handled specially.
    # So everything is kind of different.
    # Check this regex on each txt entry. If any match, the check succeeds.
    { assert_regex_match: {
      , pattern: "^keybase-site-verification=%{sig_id_medium}$"
      , from: "txt"
      , error: ["NOT_FOUND", "matching DNS entry not found"] } },
  ]]
  facebook: [[
    # Check that the claimed username has no slashes or funny business that
    # might trick us into checking a different url.
    # Facebook usernames don't actually allow any special characters, but it's
    # still possible for a malicious user to *claim* they have some slashes
    # and a question mark in their name, in the hopes that that will trick us
    # into hitting a totally unrelated URL. Guard against that happening by
    # checking for special characters in the claimed name.
    { assert_regex_match: {
      , pattern: "^[a-zA-Z0-9\\.]+$"
      , from: "username_service"
      , error: ["BAD_USERNAME", "Invalid characters in username '%{username_service}'"] } },

    # Check the provided url and extract the username and path.
    # Accept either mobile or desktop urls. The fetched url will be rewritten later.
    # We want to be strict about the structure of the url.
    # No query parameters, no unexpected characters in the post ID.
    { regex_capture: {
      , pattern: "^https://(m|www)\\.facebook\\.com/([^/]*)/posts/([0-9]+)$"
      , from: "hint_url"
      , into: ["unused1", "username_from_url", "post_id"]
      , error: ["BAD_API_URL", "Bad hint from server; URL should start with 'https://m.facebook.com/%{username_service}/posts/', got '%{hint_url}'"] } },

    # Check that the claimed username matches the url.
    # BUT this check is not sufficient: The mobile site does not care what username is passed to it.
    # It will return posts made by a user different from that in the url.
    { assert_compare: {
      , cmp: "stripdots-then-cicmp"
      , a: "username_from_url"
      , b: "username_service"
      , error: ["BAD_API_URL", "Bad hint from server; username in URL should match '%{username_service}', received '%{username_from_url}'"] } },

    # Create the mobile url using the validated username and (not validated) post id.
    { fill: {
      , with: "https://m.facebook.com/%{username_from_url}/posts/%{post_id}"
      , into: "our_url" } },
    { fetch: {
      , kind: "html"
      , from: "our_url" } },

    { selector_css: {
      , selectors: ["#m_story_permalink_view", 0, "h3", 1]
      , into: "post_text"
      , error: ["FAILED_PARSE", "Could not find post text in Facebook's response"] } },
    { whitespace_normalize: {
      , from: "post_text"
      , into: "post_text_nw" } },

    # Check the post text for username and sig
    { regex_capture: {
      , pattern: "^Verifying myself: I am (\\S+) on Keybase.io. (\\S+)$"
      , from: "post_text_nw"
      , into: ["username_from_post", "sig_from_post"]
      , error: ["TEXT_NOT_FOUND", "Could not find Verifying myself: I am %{username_keybase} on Keybase.io. (%{sig_id_medium})"] } },
    # Check username in post text
    { assert_compare: {
      , cmp: "cicmp"
      , a: "username_from_post"
      , b: "username_keybase"
      , error: ["BAD_USERNAME", "Wrong keybase username in post '%{username_from_post}' should be '%{username_keybase}'"] } },
    # Check the sig id in the post text
    { assert_compare: {
      , cmp: "exact"
      , a: "sig_id_medium"
      , b: "sig_from_post"
      , error: ["BAD_SIGNATURE", "Could not find sig; '%{sig_from_post}' != '%{sig_id_medium}'"] } },


    # Check the username in the href of the "Join" button under "Foo Barzum is on Facebook. To connect with Foo, join Facebook today."
    # Checking for the correct username is essential here. We rely on this
    # check to prove that the user in question actually wrote the post. (Note
    # that the m-site does *not* enforce this part of the URL. Only the
    # desktop site does.)
    { selector_css: {
      , selectors: ["#mobile_login_bar", 0, "a", 0]
      , attr: "href"
      , into: "join_href"
      , error: ["CONTENT_FAILURE", "Could not find 'Join' link"] } },
    # The href once looked like this for the facebook username "mike.maxim":
    # /r.php?next=https%3A%2F%2Fm.facebook.com%2Fmike.maxim%2Fposts%2F10154017925505382&cid=104&rs=10&rid=654595381
    { regex_capture: {
      , pattern: "^/r\\.php\\?next=https.*facebook.com%2F([^/]*)%2Fposts.*$"
      , from: "join_href"
      , into: ["username_from_join_href"]
      , error: ["FAILED_PARSE", "Could not interpret 'Join' link"] } },
    { assert_compare: {
      , cmp: "stripdots-then-cicmp"
      , a: "username_from_join_href"
      , b: "username_service"
      , error: ["CONTENT_FAILURE", "Bad hint from server; username in URL should match '%{username_service}', received '%{username_from_url}'"] } },
  ]]
  github: [[
    # validate url and extract username
    { regex_capture: {
      , pattern: "^https://gist\\.github(?:usercontent)?\\.com/([^/]*)/.*$"
      , from: "hint_url"
      , into: ["username_from_url"]
      , error: ["BAD_API_URL", "Bad hint from server; URL should start with either https://gist.github.com OR https://gist.githubusercontent.com"] } },
    { assert_compare: {
      , cmp: "cicmp"
      , a: "username_from_url"
      , b: "username_service"
      , error: ["BAD_API_URL", "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"] } },
    { fetch: {
      , kind: "string"
      , from: "hint_url"
      , into: "gist" } },
    # find the sig
    { assert_find_base64: {
      , needle: "sig"
      , haystack: "gist" }
      , error: ["TEXT_NOT_FOUND", "Signature not found in body"] },
    # make sure gist starts with the expected content
    # to prevent malicious gists with proofs 'hidden' in them from validating
    { whitespace_normalize: {
      , from: "gist"
      , into: "gist_nw" } },
    { assert_regex_match: {
      , pattern: "^((### Verifying myself: I am (https://keybase\\.io)?/%{username_keybase} As part of this verification process, I am signing this object and posting as a gist as github user \\*%{username_service}\\*)|(### Keybase proof I hereby claim: \\* I am %{username_service} on github\\. \\* I am %{username_keybase} \\(https://keybase\\.io/%{username_keybase}\\) on keybase\\.)) .*$"
      , case_insensitive: true
      , from: "gist_nw"
      , error: ["TEXT_NOT_FOUND", "Found sig but gist begins with unexpected content"] } },
    # Disallow the two characters ![ which start an image tag
    { assert_regex_match: {
      , pattern: "^.*!\\[.*$"
      , negate : true
      , case_insensitive: true
      , from: "gist_nw"
      , error: ["CONTENT_FAILURE", "Proof gist must not contain images"] } },
  ]]
  hackernews: [[
    # validate url and extract username
    { regex_capture: {
      , pattern: "^https://hacker-news\\.firebaseio\\.com/v0/user/([^/]+)/about.json$"
      , from: "hint_url"
      , into: ["username_from_url"]
      , error: ["BAD_API_URL", "Bad hint from server; URL should match https://hacker-news.firebaseio.com/v0/user/%{username_service}/about.json"] } },
    { assert_compare: {
      , cmp: "cicmp"
      , a: "username_from_url"
      , b: "username_service"
      , error: ["BAD_API_URL", "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"] } },
    { fetch: {
      , kind: "string"
      , from: "hint_url"
      , into: "profile" } },
    { assert_regex_match: {
      , pattern: "^.*%{sig_id_medium}.*$"
      , from: "profile"
      , error: ["TEXT_NOT_FOUND", "Posted text does not include signature '%{sig_id_medium}'"] } },
  ]]
  reddit: [[
    # validate the url
    { regex_capture: {
      , pattern: "^https://www\\.reddit\\.com/r/([^/]+)/(.*)/(\\.json)$"
      , from: "hint_url"
      , into: ["subreddit_from_url", "path_remainder", "json_trailer"]
      , error: ["BAD_API_URL", "URL should start with 'https://www.reddit.com/r/keybaseproofs'"] } },
    { assert_regex_match: {
      , pattern: "^keybaseproofs$"
      , case_insensitive: true
      , from: "subreddit_from_url"
      , error: ["BAD_API_URL", "URL contained wrong subreddit '%{subreddit_from_url}' != 'keybaseproofs'"] } },
    # Create the old.reddit.com url to fetch from
    { fill: {
      , with: "https://old.reddit.com/r/%{subreddit_from_url}/%{path_remainder}/"
      , into: "our_url" } },
    { fetch: {
      , from: "our_url"
      , kind: "html" } },
    # Select the submission itself: on an old.reddit.com comments page the post
    # is the single `link self` thing, while comments are separate `comment`
    # things. We read its fullname and the t3_ check below confirms it really is
    # a submission (t3_) and not a comment (t1_), so we fail closed if the markup
    # ever changes.
    { selector_css: {
      , selectors: ["div.thing.link.self"]
      , attr: "data-fullname"
      , into: "post_id"
      , error: ["FAILED_PARSE", "Could not find post ID"] } },
    { assert_regex_match: {
      , pattern: "^t3_.+$"
      , from: "post_id"
      , error: ["CONTENT_FAILURE", "Wanted an entry ID with 't3' prefix but got '%{post_id}'"] } },
    # check the subreddit
    { selector_css: {
      , selectors: ["div.thing.link.self"]
      , attr: "data-subreddit"
      , into: "post_subreddit"
      , error: ["FAILED_PARSE", "Could not find post subreddit"] } },
    { assert_regex_match: {
      , pattern: "^keybaseproofs$"
      , case_insensitive: true
      , from: "post_subreddit"
      , error: ["CONTENT_FAILURE", "Wrong subreddit %{post_subreddit}"] } },
    # check the author
    { selector_css: {
      , selectors: ["div.thing.link.self"]
      , attr: "data-author"
      , into: "post_author"
      , error: ["FAILED_PARSE", "Could not find post author"] } },
    { assert_compare: {
      , cmp: "cicmp"
      , a: "post_author"
      , b: "username_service"
      , error: ["BAD_USERNAME", "Bad post author; wanted '%{username_service}' but got '%{post_author}'"] } },
    # check the title
    { selector_css: {
      , selectors: ["html", "head", "title", { contents: true }]
      , into: "title"
      , error: ["TITLE_NOT_FOUND", "Could not find title in HTML"] } },
    { assert_regex_match: {
      , pattern: "^.*%{sig_id_medium}.*$"
      , multiline: true
      , from: "title"
      , error: ["TITLE_NOT_FOUND", "Missing signature ID (%{sig_id_medium}) in post title '%{title}'"] } },
    # Grab the rendered post body. The signature lives in <code>/<pre> blocks
    # whose newlines are preserved in the HTML, so pulling the whole usertext-body
    # text into one register is enough to recover a multi-line PGP signature
    # (see max's proof) as well as single-line NaCl ones.
    { selector_css: {
      , selectors: ["div.thing.link.self", "div.usertext-body"]
      , into: "post_body"
      , error: ["CONTENT_MISSING", "Could not find post body"] } },
    # Finally, find the signature in the post body.
    { assert_find_base64: {
      , needle: "sig"
      , haystack: "post_body"
      , error: ["TEXT_NOT_FOUND", "Signature not found in body"] } },
  ]]
  rooter: [[
    # URL validation.
    { assert_regex_match: {
      , pattern: "^https?://[\\w:_\\-\\.]+/_/api/1\\.0/rooter/%{username_service}/.*$"
      , case_insensitive: true
      , from: "hint_url" } },
    # rooter is special cased by the interpreter to hit the api server
    { fetch: {
      , from: "hint_url"
      , kind: "json" } },
    { selector_json: {
      , selectors: ["status", "name"]
      , into: "name" } },
    { assert_regex_match: {
      , pattern: "^ok$"
      , case_insensitive: true
      , from: "name" } },
    { selector_json: {
      , selectors: ["toot", "post"]
      , into: "post" } },
    { assert_regex_match: {
      , pattern: "^.*%{sig_id_medium}.*$"
      , from: "post" } },
  ]]
  twitter: [[
    # Twitter verification is a json fetch and two checks.
    # One that the correct user posted the tweet according to author_url.
    # And another that the proof hash is in the tweet text.

    # validate url and extract username
    { regex_capture: {
      , pattern: "^https://(?:twitter|x)\\.com/([^/]+)/status/(\\d+)(.*)$"
      , from: "hint_url"
      , into: ["username_from_url", "tweet_id", "remainder"]
      , error: ["BAD_API_URL", "Bad hint from server; URL should start with 'https://twitter.com/%{username_service}/' or 'https://x.com/%{username_service}/'"] } },
    { assert_compare: {
      , cmp: "cicmp"
      , a: "username_from_url"
      , b: "username_service"
      , error: ["BAD_API_URL", "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"] } },
    # Create the 'oembed' url to fetch from
    { fill: {
      , with: "https://api.twitter.com/1/statuses/oembed.json?id=%{tweet_id}"
      , into: "our_url" } },
    # url validation passed
    { fetch: {
      , from: "our_url"
      , kind: "json" } },

    # Check the author.
    # Use author_url. Don't use author_name because that is sometimes a full name like 'Max Krohn'
    { selector_json: {
      , selectors: ["author_url"]
      , into: "author_url"
      , error: ["CONTENT_MISSING", "Could not find 'author_url' in json"] } },
    { regex_capture: {
      , pattern: "^https://(?:twitter|x)\\.com/(.+)$"
      , from: "author_url"
      , into: ["author_username"]
      , error: ["CONTENT_MISSING", "Could not capture username from author_url"] } },
    # Check the username in the tweet. Case insensitive.
    { assert_compare: {
      , cmp: "cicmp"
      , a: "author_username"
      , b: "username_service"
      , error: ["BAD_USERNAME", "Bad post authored: wanted '%{username_service}' but got '%{author_username}'"] } },

    { selector_json: {
      , selectors: ["html"]
      , into: "tweet_contents"
      , error: ["CONTENT_MISSING", "Missing 'tweet_html' in json"] } },
    { whitespace_normalize: {
      , from: "tweet_contents"
      , into: "tweet_contents_nw" } },
    # Get username and sig from tweet.
    # We would like to assert that "keybase" or "keybase.io" appears in the contents, but cannot.
    # Because for some tweets twitter returns t.co-ified links with no "keybase.io" label:
    #   "Verifying myself: I am xinyuzhao on <a href=\"https://t.co/5kAXqNmZtI\">https://t.co/5kAXqNmZtI</a>. n2odH86L6_FXLRk4ed6_HvYkhR7BGH-qGBQ_ /"
    { regex_capture: {
      , pattern: "^.*Verifying myself: I am ([A-Za-z0-9_]+) on (?:(?:https?://)?Keybase\\.io|<a href=\"https://t\\.co/[A-Za-z0-9_-]+\">https://t\\.co/[A-Za-z0-9_-]+</a>)[\\s\\p{Zs}]*\\. (\\S+) */.*$"
      , from: "tweet_contents_nw"
      , into: ["username_from_tweet_contents", "sig_from_tweet_contents"]
      , error: ["DELETED", "Could not find 'Verifying myself: I am %{username_keybase} on Keybase.io. %{sig_id_short}'"] } },
    # Check username in tweet body
    { assert_compare: {
      , cmp: "cicmp"
      , a: "username_from_tweet_contents"
      , b: "username_keybase"
      , error: ["BAD_USERNAME", "Wrong username in tweet '%{username_from_tweet_contents}' should be '%{username_keybase}'"] } },
    # Check the sig id in the tweet.
    { assert_regex_match: {
      , pattern: "^%{sig_id_short}$"
      , from: "sig_from_tweet_contents"
      , error: ["TEXT_NOT_FOUND", "Could not find sig '%{sig_from_tweet_contents}' != '%{sig_id_short}'"] } },
  ]]
  generic_web_site: [[
    # URL validation. Must be HTTP or HTTPS. Must be a known path from the proof domain.
    { assert_regex_match: {
      , pattern: "^%{protocol}://%{hostname}/(?:\\.well-known/keybase\\.txt|keybase\\.txt)$"
      , from: "hint_url"
      , error: ["BAD_API_URL", "Bad hint from server; didn't recognize API url: \"%{hint_url}\""]} },
    { fetch: {
      , kind: "string"
      , from: "hint_url"
      , into: "blob" } },
    # Verify and find the sig.
    { assert_find_base64: {
      , needle: "sig"
      , haystack: "blob"
      , error: ["TEXT_NOT_FOUND", "Signature not found in body"] } },
  ]]