☰

Twitter Direct

Remove t.co tracking links from Twitter

当前为 2021-05-13 提交的版本，查看 最新版本。

  1. // ==UserScript==
  2. // @name          Twitter Direct
  3. // @description   Remove t.co tracking links from Twitter
  4. // @author        chocolateboy
  5. // @copyright     chocolateboy
  6. // @version       1.8.0
  7. // @namespace     https://github.com/chocolateboy/userscripts
  8. // @license       GPL
  9. // @include       https://twitter.com/
  10. // @include       https://twitter.com/*
  11. // @include       https://mobile.twitter.com/
  12. // @include       https://mobile.twitter.com/*
  13. // @require       https://unpkg.com/@chocolateboy/uncommonjs@3.1.2/dist/polyfill.iife.min.js
  14. // @require       https://unpkg.com/get-wild@1.5.0/dist/index.umd.min.js
  15. // @require       https://unpkg.com/gm-compat@1.1.0/dist/index.iife.min.js
  16. // @require       https://unpkg.com/just-safe-set@2.2.1/index.js
  17. // @run-at        document-start
  18. // ==/UserScript==
  19.  
  20. /*
  21.  * a pattern which matches the content-type header of responses we scan for
  22.  * URLs: "application/json" or "application/json; charset=utf-8"
  23.  */
  24. const CONTENT_TYPE = /^application\/json\b/
  25.  
  26. /*
  27.  * the minimum size (in bytes) of documents we deem to be "not small"
  28.  *
  29.  * we log (to the console) misses (i.e. no URLs ever found/replaced) in
  30.  * documents whose size is greater than or equal to this value
  31.  *
  32.  * if we keep failing to find URLs in large documents, we may be able to speed
  33.  * things up by blacklisting them, at least in theory
  34.  *
  35.  * (in practice, URL data is optional in most of the matched document types
  36.  * (contained in arrays that can be empty), so an absence of URLs doesn't
  37.  * necessarily mean URL data will never be included...)
  38.  */
  39. const LOG_THRESHOLD = 1024
  40.  
  41. /*
  42.  * an immutable array used in various places as a way to indicate "no values".
  43.  * static to avoid unnecessary allocations.
  44.  */
  45. const NONE = []
  46.  
  47. /*
  48.  * used to keep track of which queries (don't) have matching URIs and which URIs
  49.  * (don't) have matching queries
  50.  */
  51. const STATS = { root: {}, uri: {} }
  52.  
  53. /*
  54.  * the domain intercepted links are routed through
  55.  *
  56.  * not all links are intercepted. exceptions include links to twitter (e.g.
  57.  * https://twitter.com) and card URIs (e.g. card://123456)
  58.  */
  59. const TRACKING_DOMAIN = 't.co'
  60.  
  61. /*
  62.  * a pattern which matches the domain(s) we expect data (JSON) to come from.
  63.  * responses which don't come from a matching domain are ignored.
  64.  */
  65. const TWITTER_API = /^(?:(?:api|mobile)\.)?twitter\.com$/
  66.  
  67. /*
  68.  * default locations to search for URL metadata (arrays of objects) within tweet
  69.  * nodes
  70.  */
  71. const TWEET_PATHS = [
  72.     'entities.media',
  73.     'entities.urls',
  74.     'extended_entities.media',
  75.     'extended_entities.urls',
  76. ]
  77.  
  78. /*
  79.  * default locations to search for URL metadata (arrays of objects) within
  80.  * user/profile nodes
  81.  */
  82. const USER_PATHS = [
  83.     'entities.description.urls',
  84.     'entities.url.urls',
  85. ]
  86.  
  87. /*
  88.  * a router which matches URIs (pathnames) to queries. each query contains a
  89.  * root path (required) and some additional options which specify the locations
  90.  * under the root path to substitute URLs in.
  91.  *
  92.  * implemented as an array of pairs with URI-pattern keys (string(s) or
  93.  * regexp(s)) and one or more queries as the value. if a query is a string
  94.  * (path), it is converted into an object with the path as its `root`
  95.  * property.
  96.  *
  97.  * options:
  98.  *
  99.  *   - root (required): a path (string or array of steps) into the document
  100.  *     under which to begin searching
  101.  *
  102.  *   - collect (default: Object.values): a function which takes a root node and
  103.  *     turns it into an array of context nodes to scan for URL data
  104.  *
  105.  *   - scan (default: USER_PATHS): an array of paths to probe for arrays of
  106.  *     { url, expanded_url } pairs in a context node
  107.  *
  108.  *   - targets (default: NONE): an array of paths to standalone URLs (URLs that
  109.  *     don't have an accompanying expansion), e.g. for URLs in cards embedded in
  110.  *     tweets. these URLs are replaced by expanded URLs gathered during
  111.  *     preceding scans.
  112.  *
  113.  *     target paths can point directly to a URL node (string), or to an array
  114.  *     or plain object, in which case the URL is located inside the array/object
  115.  *     and replaced
  116.  *
  117.  *     if a target path is an object containing a { url: path, expanded_url: path }
  118.  *     pair, the URL is expanded directly in the same way as scanned paths.
  119.  */
  120. const MATCH = [
  121.     [
  122.         /\/Bookmarks$/, [
  123.             'data.bookmark_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy.user_refs.*.legacy',
  124.             'data.bookmark_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.core.user.legacy',
  125.             {
  126.                 root: 'data.bookmark_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy',
  127.                 scan: TWEET_PATHS,
  128.                 targets: [
  129.                     {
  130.                         url: 'quoted_status_permalink.url',
  131.                         expanded_url: 'quoted_status_permalink.expanded',
  132.                     }
  133.                 ]
  134.             },
  135.             {
  136.                 root: 'data.bookmark_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy',
  137.  
  138.                 // just expand the URLs in the specified locations within the
  139.                 // card; there's no user or tweet data under this root
  140.                 scan: NONE,
  141.  
  142.                 targets: ['binding_values', 'url'],
  143.             },
  144.         ],
  145.     ],
  146.     [
  147.         /\/Conversation$/, [
  148.             'data.conversation_timeline.instructions.*.moduleItems.*.item.itemContent.tweet.core.user.legacy',
  149.             'data.conversation_timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.core.user.legacy',
  150.             {
  151.                 root: 'data.conversation_timeline.instructions.*.moduleItems.*.item.itemContent.tweet.legacy',
  152.                 scan: TWEET_PATHS,
  153.                 targets: ['card.binding_values', 'card.url'],
  154.             },
  155.             {
  156.                 root: 'data.conversation_timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.legacy',
  157.                 scan: TWEET_PATHS,
  158.                 targets: ['card.binding_values', 'card.url'],
  159.             },
  160.         ]
  161.     ],
  162.     [
  163.         /\/Favoriters$/,
  164.         'data.favoriters_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy',
  165.     ],
  166.     [
  167.         /\/Following$/,
  168.         'data.user.following_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy',
  169.     ],
  170.     [
  171.         /\/Followers$/,
  172.         'data.user.followers_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy',
  173.     ],
  174.     [
  175.         /\/FollowersYouKnow$/,
  176.         'data.user.friends_following_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy',
  177.     ],
  178.     [
  179.         // "List" page, e.g. /i/api/graphql/abcd1234/ListLatestTweetsTimeline
  180.         /\/ListLatestTweetsTimeline$/, [
  181.             'data.list.tweets_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.core.user.legacy',
  182.             {
  183.                 root: 'data.list.tweets_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy',
  184.                 scan: TWEET_PATHS,
  185.             },
  186.             {
  187.                 root: 'data.list.tweets_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.legacy',
  188.                 scan: TWEET_PATHS,
  189.             },
  190.             {
  191.                 root: 'data.list.tweets_timeline.timeline.instructions.*.entries.*.content.itemContent.tweet',
  192.                 scan: NONE,
  193.                 targets: [
  194.                     'card.legacy.binding_values',
  195.                     'card.legacy.url',
  196.                     'legacy.retweeted_status.card.legacy.binding_values',
  197.                     'legacy.retweeted_status.card.legacy.url',
  198.                 ],
  199.             },
  200.         ]
  201.     ],
  202.     [
  203.         // "Likes" page, e.g. /i/api/graphql/abcd1234/Likes
  204.         /\/Likes$/, [
  205.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.core.user.legacy',
  206.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.quoted_status.card.legacy.user_refs.*.legacy',
  207.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.quoted_status.core.user.legacy',
  208.             {
  209.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.quoted_status.legacy',
  210.                 scan: TWEET_PATHS,
  211.             },
  212.             {
  213.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy',
  214.                 scan: TWEET_PATHS,
  215.                 targets: [
  216.                     {
  217.                         url: 'quoted_status_permalink.url',
  218.                         expanded_url: 'quoted_status_permalink.expanded',
  219.                     },
  220.                 ],
  221.             },
  222.             {
  223.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet',
  224.                 scan: NONE,
  225.                 targets: [
  226.                     'card.legacy.binding_values',
  227.                     'card.legacy.url',
  228.                     'quoted_status.card.legacy.binding_values',
  229.                     'quoted_status.card.legacy.url',
  230.                 ],
  231.             },
  232.         ],
  233.     ],
  234.     [
  235.         /\/ListMembers$/,
  236.         'data.list.members_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy'
  237.     ],
  238.     [
  239.         /\/ListSubscribers$/,
  240.         'data.list.subscribers_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy',
  241.     ],
  242.     [
  243.         /\/Retweeters/,
  244.         'data.retweeters_timeline.timeline.instructions.*.entries.*.content.itemContent.user.legacy'
  245.     ],
  246.     [
  247.         /\/TweetDetail$/, [
  248.             'data.threaded_conversation_with_injections.instructions.*.entries.*.content.itemContent.tweet.card.legacy.user_refs.*.legacy',
  249.             {
  250.                 root: 'data.threaded_conversation_with_injections.instructions.*.entries.*.content.itemContent.tweet.legacy',
  251.                 scan: TWEET_PATHS,
  252.             },
  253.             {
  254.                 root: 'data.threaded_conversation_with_injections.instructions.*.entries.*.content.items.*.item.itemContent.tweet.legacy',
  255.                 scan: TWEET_PATHS,
  256.             },
  257.             {
  258.                 root: 'data.threaded_conversation_with_injections.instructions.*.entries.*.content.items.*.item.itemContent.tweet.core.user.legacy',
  259.                 targets: ['url'],
  260.             },
  261.             {
  262.                 root: 'data.threaded_conversation_with_injections.instructions.*.entries.*.content.itemContent.tweet.card.legacy',
  263.                 scan: NONE,
  264.                 targets: ['binding_values', 'url', 'user_refs.*.legacy.url'],
  265.             }
  266.         ],
  267.     ],
  268.     [
  269.         // used for hovercard data
  270.         /\/UserByScreenName$/, {
  271.             root: 'data.user.legacy',
  272.             collect: Array.of,
  273.         }
  274.     ],
  275.     [
  276.         /\/UserByScreenNameWithoutResults$/, {
  277.             root: 'data.user.legacy',
  278.             collect: Array.of,
  279.         },
  280.     ],
  281.     [
  282.         // e.g. /i/api/graphql/abcd1234/UserMedia
  283.         /\/UserMedia$/, [
  284.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.core.user.legacy',
  285.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.quoted_status.card.legacy.user_refs.*.legacy',
  286.             {
  287.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.quoted_status.legacy',
  288.                 scan: TWEET_PATHS,
  289.             },
  290.             {
  291.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy',
  292.                 scan: TWEET_PATHS,
  293.                 targets: [
  294.                     {
  295.                         url: 'quoted_status_permalink.url',
  296.                         expanded_url: 'quoted_status_permalink.expanded',
  297.                     },
  298.                 ],
  299.             },
  300.             {
  301.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy.user_refs.*.legacy',
  302.                 targets: ['url'],
  303.             },
  304.         ]
  305.     ],
  306.     [
  307.         // e.g. /i/api/graphql/abcd1234/UserTweets
  308.         /\/UserTweets$/, [
  309.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.core.user.legacy',
  310.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.core.user.legacy',
  311.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.user.legacy',
  312.             'data.user.result.timeline.timeline.instructions.*.entry.content.itemContent.tweet.core.user.legacy',
  313.             {
  314.                 root: 'data.user.legacy',
  315.                 collect: Array.of,
  316.             },
  317.             {
  318.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.legacy',
  319.                 scan: TWEET_PATHS,
  320.             },
  321.             {
  322.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.quoted_status.legacy',
  323.                 scan: TWEET_PATHS,
  324.             },
  325.             {
  326.                 root: 'data.user.result.timeline.timeline.instructions.*.entry.content.itemContent.tweet.legacy',
  327.                 scan: TWEET_PATHS,
  328.             },
  329.             {
  330.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.quoted_status.card.legacy',
  331.                 targets: ['binding_values', 'url'],
  332.             },
  333.             {
  334.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.card.legacy.user_refs.*.legacy',
  335.                 targets: ['url'],
  336.             },
  337.             {
  338.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.quoted_status.card.legacy.user_refs.*.legacy',
  339.                 targets: ['url'],
  340.             },
  341.             {
  342.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.card.legacy.user_refs.*.legacy',
  343.                 targets: ['url'],
  344.             },
  345.             {
  346.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.core.user.legacy',
  347.                 targets: ['url'],
  348.             },
  349.             {
  350.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy.user_refs.*.legacy',
  351.                 targets: ['url'],
  352.             },
  353.             {
  354.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy',
  355.                 scan: TWEET_PATHS,
  356.                 targets: [
  357.                     {
  358.                         url: 'quoted_status_permalink.url',
  359.                         expanded_url: 'quoted_status_permalink.expanded',
  360.                     },
  361.                 ],
  362.             },
  363.             {
  364.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.legacy',
  365.                 scan: TWEET_PATHS,
  366.                 targets: [
  367.                     {
  368.                         url: 'quoted_status_permalink.url',
  369.                         expanded_url: 'quoted_status_permalink.expanded',
  370.                     },
  371.                 ],
  372.             },
  373.             {
  374.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy',
  375.                 scan: NONE,
  376.                 targets: ['binding_values', 'url'],
  377.             },
  378.             {
  379.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.card.legacy',
  380.                 scan: NONE,
  381.                 targets: ['binding_values', 'url'],
  382.             },
  383.             {
  384.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.card.legacy',
  385.                 scan: NONE,
  386.                 targets: ['binding_values', 'url'],
  387.             },
  388.         ]
  389.     ],
  390.     [
  391.         // e.g. /i/api/graphql/abcd1234/UserTweetsAndReplies
  392.         /\/UserTweetsAndReplies$/, [
  393.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.core.user.legacy',
  394.             'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.core.user.legacy',
  395.             'data.user.result.timeline.timeline.instructions.*.entry.content.itemContent.tweet.core.user.legacy',
  396.             {
  397.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.legacy',
  398.                 scan: TWEET_PATHS,
  399.             },
  400.             {
  401.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy',
  402.                 scan: TWEET_PATHS,
  403.                 targets: [
  404.                     {
  405.                         url: 'quoted_status_permalink.url',
  406.                         expanded_url: 'quoted_status_permalink.expanded',
  407.                     },
  408.                 ]
  409.             },
  410.             {
  411.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.legacy',
  412.                 scan: TWEET_PATHS,
  413.                 targets: [
  414.                     {
  415.                         url: 'quoted_status_permalink.url',
  416.                         expanded_url: 'quoted_status_permalink.expanded',
  417.                     },
  418.                 ],
  419.             },
  420.             {
  421.                 root: 'data.user.result.timeline.timeline.instructions.*.entry.content.itemContent.tweet.legacy',
  422.                 scan: TWEET_PATHS,
  423.             },
  424.             {
  425.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.core.user.legacy',
  426.                 targets: ['url'],
  427.             },
  428.             {
  429.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.items.*.item.itemContent.tweet.card.legacy',
  430.                 scan: NONE,
  431.                 targets: ['binding_values', 'url'],
  432.             },
  433.             {
  434.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy',
  435.                 scan: NONE,
  436.                 targets: ['binding_values', 'url'],
  437.             },
  438.             {
  439.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.legacy.retweeted_status.card.legacy',
  440.                 scan: NONE,
  441.                 targets: ['binding_values', 'url'],
  442.             },
  443.             {
  444.                 root: 'data.user.result.timeline.timeline.instructions.*.entries.*.content.itemContent.tweet.card.legacy.user_refs.*.legacy',
  445.                 targets: ['url'],
  446.             },
  447.         ]
  448.     ],
  449.     [
  450.         // DMs
  451.         // e.g. '/1.1/dm/inbox_initial_state.json' and '/1.1/dm/user_updates.json'
  452.         /\/(?:inbox_initial_state|user_updates)\.json$/, {
  453.             root: 'inbox_initial_state.entries.*.message.message_data',
  454.             scan: TWEET_PATHS,
  455.             targets: ['attachment.card.binding_values', 'attachment.card.url'],
  456.         }
  457.     ],
  458.     [
  459.         // e.g. '/1.1/friends/following/list.json',
  460.         /\/list\.json$/, {
  461.             root: 'users.*'
  462.         },
  463.     ],
  464.     [
  465.         // e.g. '/1.1/users/lookup.json',
  466.         /\/lookup\.json$/, {
  467.             root: NONE, // the document itself (an array of users) is the root
  468.         }
  469.     ],
  470.     [
  471.         // "Who to follow"
  472.         // e.g. '/1.1/users/recommendations.json'
  473.         /\/recommendations\.json$/, {
  474.             root: '*.user',
  475.         }
  476.     ],
  477. ]
  478.  
  479. /*
  480.  * a single { pattern => queries } pair for the router which is run against all
  481.  * documents
  482.  */
  483. const WILDCARD = [
  484.     /./,
  485.     [
  486.         'globalObjects.users',
  487.         'globalObjects.tweets.*.card.users',
  488.         {
  489.             root: 'globalObjects.tweets',
  490.             scan: TWEET_PATHS,
  491.             targets: [
  492.                 {
  493.                     url: 'card.binding_values.website_shortened_url.string_value',
  494.                     expanded_url: 'card.binding_values.website_url.string_value',
  495.                 },
  496.                 'card.binding_values',
  497.                 'card.url',
  498.             ],
  499.         },
  500.     ]
  501. ]
  502.  
  503. /*
  504.  * a custom version of get-wild's `get` function which uses a simpler/faster
  505.  * path parser since we don't use the extended syntax
  506.  */
  507. const get = exports.getter({ split: '.' })
  508.  
  509. /*
  510.  * a helper function which returns true if the supplied value is a plain object,
  511.  * false otherwise
  512.  */
  513. const isPlainObject = (function () {
  514.     const toString = {}.toString
  515.     // only used with JSON data, so we don't need this to be foolproof
  516.     return value => toString.call(value) === '[object Object]'
  517. })()
  518.  
  519. /*
  520.  * a helper function which iterates over the supplied iterable, filtering out
  521.  * missing (undefined) values.
  522.  *
  523.  * this is done in one pass (rather than map + filter) as there may potentially
  524.  * be dozens or even hundreds of values, e.g. contexts (tweet/user objects)
  525.  * under a root node
  526.  */
  527. function eachDefined (iterable, fn) {
  528.     for (const value of iterable) {
  529.         if (value) fn(value)
  530.     }
  531. }
  532.  
  533. /**
  534.  * a helper function which returns true if the supplied URL is tracked by
  535.  * Twitter, false otherwise
  536.  */
  537. function isTracked (url) {
  538.     return (new URL(url)).hostname === TRACKING_DOMAIN
  539. }
  540.  
  541. /*
  542.  * JSON.stringify helper used to serialize stats data
  543.  */
  544. function replacer (_key, value) {
  545.     return (value instanceof Set) ? Array.from(value) : value
  546. }
  547.  
  548. /*
  549.  * a generator which returns { pattern => queries } pairs where patterns are
  550.  * strings/regexps which match a URI and queries are objects which define
  551.  * substitutions to perform in the matched document.
  552.  *
  553.  * this forms the basis of a simple "router" which tries all URI patterns until
  554.  * one matches (or none match) and then additionally performs a wildcard match
  555.  * which works on all URIs.
  556.  *
  557.  * the URI patterns are disjoint, so there's no need to try them all if one
  558.  * matches. in addition to these, some substitutions are non URI-specific, i.e.
  559.  * they work on documents that aren't matched by URI (e.g. profile.json) and
  560.  * documents that are (e.g. list.json). currently these cross-document nodes can
  561.  * be found under obj.globalObjects, so we check for the existence of that
  562.  * property before yielding these catch-all queries
  563.  */
  564. function* router (data, state) {
  565.     for (const [key, value] of MATCH) {
  566.         yield [key, value]
  567.  
  568.         if (state.matched) {
  569.             break
  570.         }
  571.     }
  572.  
  573.     if ('globalObjects' in data) {
  574.         yield WILDCARD
  575.     }
  576. }
  577.  
  578. /*
  579.  * a helper class which implements document-specific (MATCH) and generic
  580.  * (WILDCARD) URL substitutions in nodes (subtrees) within a JSON-formatted
  581.  * document returned by the Twitter API.
  582.  *
  583.  * a transformer is instantiated for each query and its methods are passed a
  584.  * context (node within the document tree) and the value of an option from the
  585.  * query, e.g. the `scan` option is handled by the `_scan` method and the
  586.  * `targets` option is processed by the `_assign` method
  587.  */
  588. class Transformer {
  589.     constructor ({ cache, onReplace, root, uri }) {
  590.         this._cache = cache
  591.         this._onReplace = onReplace
  592.         this._root = root
  593.         this._uri = uri
  594.     }
  595.  
  596.     /*
  597.      * expand URLs in context nodes in the locations specified by the query's
  598.      * `scan` and `targets` options
  599.      */
  600.     // @ts-ignore https://github.com/microsoft/TypeScript/issues/14279
  601.     transform (contexts, scan, targets) {
  602.         // scan the context nodes for { url, expanded_url } pairs, replace
  603.         // each t.co URL with its expansion, and add the mappings to the
  604.         // cache
  605.         eachDefined(contexts, context => this._scan(context, scan))
  606.  
  607.         // do a separate pass for targets because the scan may have added more
  608.         // mappings to the cache
  609.         if (targets.length) {
  610.             eachDefined(contexts, context => this._assign(context, targets))
  611.         }
  612.     }
  613.  
  614.     /*
  615.      * scan the context node for { url, expanded_url } pairs, replace each t.co
  616.      * URL with its expansion, and add the mappings to the cache
  617.      */
  618.     _scan (context, paths) {
  619.         const { _cache: cache, _onReplace: onReplace } = this
  620.  
  621.         for (const path of paths) {
  622.             const items = get(context, path, NONE)
  623.  
  624.             for (const item of items) {
  625.                 if (item.url && item.expanded_url) {
  626.                     if (isTracked(item.url)) {
  627.                         cache.set(item.url, item.expanded_url)
  628.                         item.url = item.expanded_url
  629.                         onReplace()
  630.                     }
  631.                 } else {
  632.                     console.warn("can't find url/expanded_url pair for:", {
  633.                         uri: this._uri,
  634.                         root: this._root,
  635.                         path,
  636.                         item,
  637.                     })
  638.                 }
  639.             }
  640.         }
  641.     }
  642.  
  643.     /*
  644.      * replace URLs in the context which weren't substituted during the scan.
  645.      *
  646.      * these are either standalone URLs whose expansion we retrieve from the
  647.      * cache, or URLs whose expansion exists in the context in a location not
  648.      * covered by the scan
  649.      */
  650.     _assign (context, targets) {
  651.         for (const target of targets) {
  652.             if (isPlainObject(target)) {
  653.                 this._assignFromPath(context, target)
  654.             } else {
  655.                 this._assignFromCache(context, target)
  656.             }
  657.         }
  658.     }
  659.  
  660.     /*
  661.      * replace a short URL in the context with an expanded URL defined in the
  662.      * context, e.g. context.foo.url = context.bar.baz.expanded_url
  663.      *
  664.      * this is similar to the replacements performed during the scan, but rather
  665.      * than using a fixed set of locations/paths, the paths to the short and
  666.      * expanded URLs are supplied as a parameter
  667.      */
  668.     _assignFromPath (context, paths) {
  669.         const { url: urlPath, expanded_url: expandedUrlPath } = paths
  670.  
  671.         let url, expandedUrl
  672.  
  673.         if (
  674.                (url = get(context, urlPath))
  675.             && isTracked(url)
  676.             && (expandedUrl = get(context, expandedUrlPath))
  677.         ) {
  678.             this._cache.set(url, expandedUrl)
  679.             exports.set(context, urlPath, expandedUrl)
  680.             this._onReplace()
  681.         }
  682.     }
  683.  
  684.     /*
  685.      * pinpoint an isolated URL in the context which doesn't have a
  686.      * corresponding expansion, and replace it using the mappings collected
  687.      * during preceding scans
  688.      */
  689.     _assignFromCache (context, path) {
  690.         const node = get(context, path)
  691.  
  692.         let url
  693.  
  694.         // special-case card URLs
  695.         //
  696.         // if the target is an array or plain object, locate its target URL
  697.         // automatically. used to resolve "binding" nodes, which represent
  698.         // properties (key/value pairs) as an array or object
  699.  
  700.         if (Array.isArray(node)) {
  701.             const found = node.find(it => it?.key === 'card_url')
  702.  
  703.             if (found) {
  704.                 context = found
  705.                 path = 'value.string_value'
  706.                 url = get(context, path)
  707.             }
  708.         } else if (isPlainObject(node)) {
  709.             if (node.card_url) {
  710.                 context = node
  711.                 path = 'card_url.string_value'
  712.                 url = get(context, path)
  713.             }
  714.         } else {
  715.             url = node
  716.         }
  717.  
  718.         if (typeof url === 'string' && isTracked(url)) {
  719.             const expandedUrl = this._cache.get(url)
  720.  
  721.             if (expandedUrl) {
  722.                 exports.set(context, path, expandedUrl)
  723.                 this._onReplace()
  724.             } else {
  725.                 console.warn(`can't find expanded URL for ${url} in ${this._uri}`)
  726.             }
  727.         }
  728.     }
  729. }
  730.  
  731. /*
  732.  * replace t.co URLs with the original URL in all locations in the document
  733.  * which contain URLs
  734.  *
  735.  * returns the number of substituted URLs
  736.  */
  737. function transform (data, uri) {
  738.     let count = 0 // keep track of and return the number of expanded URLs
  739.  
  740.     if (!STATS.uri[uri]) {
  741.         STATS.uri[uri] = new Set()
  742.     }
  743.  
  744.     // t.co -> expanded URL mapping for all queries in this document. used to
  745.     // fill in isolated URLs (usually in cards) which don't have a corresponding
  746.     // `expanded_url`
  747.     const cache = new Map()
  748.  
  749.     // used to notify the router when a pattern matches so it can stop trying
  750.     // URI matches and transition to the wildcard match
  751.     const state = { matched: false }
  752.  
  753.     // an iterator which yields { pattern => queries } pairs for this document.
  754.     // if a document's URI matches the pattern, the corresponding queries
  755.     // (search + replace) are executed
  756.     const it = router(data, state)
  757.  
  758.     for (const [key, value] of it) {
  759.         const uris = NONE.concat(key) // coerce to an array
  760.         const queries = NONE.concat(value)
  761.         const match = uris.some(want => {
  762.             return (typeof want === 'string') ? (uri === want) : want.test(uri)
  763.         })
  764.  
  765.         if (match) {
  766.             // stop matching URIs after this and switch to the wildcard queries
  767.             state.matched = true
  768.         } else {
  769.             // try the next URI pattern, or switch to the wildcard queries if
  770.             // there are no more patterns to match against
  771.             continue
  772.         }
  773.  
  774.         for (const $query of queries) {
  775.             const query = isPlainObject($query) ? $query : { root: $query }
  776.             const { root: rootPath } = query
  777.  
  778.             if (!STATS.root[rootPath]) {
  779.                 STATS.root[rootPath] = new Set()
  780.             }
  781.  
  782.             const root = get(data, rootPath)
  783.  
  784.             // might be an array (e.g. lookup.json)
  785.             if (!root || typeof root !== 'object') {
  786.                 continue
  787.             }
  788.  
  789.             const {
  790.                 collect = Object.values,
  791.                 scan = USER_PATHS,
  792.                 targets = NONE,
  793.             } = query
  794.  
  795.             const updateStats = () => {
  796.                 ++count
  797.                 STATS.uri[uri].add(rootPath)
  798.                 STATS.root[rootPath].add(uri)
  799.             }
  800.  
  801.             const contexts = collect(root)
  802.  
  803.             const transformer = new Transformer({
  804.                 cache,
  805.                 onReplace: updateStats,
  806.                 root: rootPath,
  807.                 uri,
  808.             })
  809.  
  810.             // @ts-ignore https://github.com/microsoft/TypeScript/issues/14279
  811.             transformer.transform(contexts, scan, targets)
  812.         }
  813.     }
  814.  
  815.     return count
  816. }
  817.  
  818. /*
  819.  * replacement for Twitter's default handler for XHR requests. we transform the
  820.  * response if it's a) JSON and b) contains URL data; otherwise, we leave it
  821.  * unchanged
  822.  */
  823. function onResponse (xhr, uri) {
  824.     const contentType = xhr.getResponseHeader('Content-Type')
  825.  
  826.     if (!CONTENT_TYPE.test(contentType)) {
  827.         return
  828.     }
  829.  
  830.     const url = new URL(uri)
  831.  
  832.     // exclude e.g. the config-<date>.json file from pbs.twimg.com, which is the
  833.     // second biggest document (~500K) after home_latest.json (~700K)
  834.     if (!TWITTER_API.test(url.hostname)) {
  835.         return
  836.     }
  837.  
  838.     const json = xhr.responseText
  839.     const size = json.length
  840.  
  841.     // fold URIs which differ only in the user ID, e.g.:
  842.     // /2/timeline/profile/1234.json -> /2/timeline/profile.json
  843.     const path = url.pathname.replace(/\/\d+\.json$/, '.json')
  844.  
  845.     let data
  846.  
  847.     try {
  848.         data = JSON.parse(json)
  849.     } catch (e) {
  850.         console.error(`Can't parse JSON for ${uri}:`, e)
  851.         return
  852.     }
  853.  
  854.     const oldStats = JSON.stringify(STATS, replacer)
  855.     const count = transform(data, path)
  856.  
  857.     if (!count) {
  858.         if (STATS.uri[path].size === 0 && size >= LOG_THRESHOLD) {
  859.             console.debug(`no replacements in ${path} (${size} B)`)
  860.         }
  861.  
  862.         return
  863.     }
  864.  
  865.     const descriptor = { value: JSON.stringify(data) }
  866.     const clone = GMCompat.export(descriptor)
  867.  
  868.     GMCompat.unsafeWindow.Object.defineProperty(xhr, 'responseText', clone)
  869.  
  870.     const newStats = JSON.stringify(STATS, replacer)
  871.  
  872.     if (newStats !== oldStats) {
  873.         const replacements = 'replacement' + (count === 1 ? '' : 's')
  874.         console.debug(`${count} ${replacements} in ${path} (${size} B)`)
  875.         console.log(JSON.parse(newStats))
  876.     }
  877. }
  878.  
  879. /*
  880.  * replace the built-in XHR#send method with our custom version which swaps in
  881.  * our custom response handler. once done, we delegate to the original handler
  882.  * (this.onreadystatechange)
  883.  */
  884. function hookXHRSend (oldSend) {
  885.     return /** @this {XMLHttpRequest} */ function send (body = null) {
  886.         const oldOnReadyStateChange = this.onreadystatechange
  887.  
  888.         this.onreadystatechange = function (event) {
  889.             if (this.readyState === this.DONE && this.responseURL && this.status === 200) {
  890.                 onResponse(this, this.responseURL)
  891.             }
  892.  
  893.             if (oldOnReadyStateChange) {
  894.                 oldOnReadyStateChange.call(this, event)
  895.             }
  896.         }
  897.  
  898.         oldSend.call(this, body)
  899.     }
  900. }
  901.  
  902. /*
  903.  * replace the default XHR#send with our custom version, which scans responses
  904.  * for tweets and expands their URLs
  905.  */
  906. const xhrProto = GMCompat.unsafeWindow.XMLHttpRequest.prototype
  907.  
  908. xhrProto.send = GMCompat.export(hookXHRSend(xhrProto.send))