# This file comes from an adaptation of rules present in
# https://github.com/webrecorder/wabac.js/blame/main/src/fuzzymatcher.js
#
# Syncing rules is done manually, based on expert knowledge, especially because in
# warc2zim we are not really fuzzy matching (searching the best entry among existing
# ones) but just rewriting to proper path.
#
# This file is in sync with content at commit 879018d5b96962df82340a9a57570bbc0fc67815
# from June 9, 2024
#
# This file should be updated at every release of warc2zim
#
# Some rules are voluntarily missing because not been tested in warc2zim yet: Twitter,
# Washington Post, WixStatic, Facebook
#
# Generic rules are also ommitted on purpose, we don't need them
#
version: "20240807_01"
fuzzyRules:
  - name: googlevideo_com
    pattern: .*googlevideo.com/(videoplayback(?=\?)).*[?&](id=[^&]+).*
    replace: youtube.fuzzy.replayweb.page/\1?\2
    tests:
      - raw_url: foobargooglevideo.com/videoplayback?id=1576&key=value
        fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576
      - raw_url: foobargooglevideo.com/videoplayback?some=thing&id=1576
        fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576
      - raw_url: foobargooglevideo.com/videoplayback?some=thing&id=1576&key=value
        fuzzified_url: youtube.fuzzy.replayweb.page/videoplayback?id=1576
      - raw_url: foobargooglevideo.com/videoplaybackandfoo?some=thing&id=1576&key=value
        unchanged: true  # videoplayback is not followed by `?`
      - raw_url: foobargoogle_video.com/videoplaybackandfoo?some=thing&id=1576&key=value
        unchanged: true  # No googlevideo.com in url
  - name: youtube_video_info
    pattern: (?:www\.)?youtube(?:-nocookie)?\.com/(get_video_info\?).*(video_id=[^&]+).*
    replace : youtube.fuzzy.replayweb.page/\1\2
    tests:
      - raw_url: www.youtube.com/get_video_info?video_id=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
      - raw_url: www.youtube.com/get_video_info?foo=bar&video_id=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
      - raw_url: www.youtube.com/get_video_info?video_id=123ah&foo=bar
        fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
      - raw_url: youtube.com/get_video_info?video_id=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
      - raw_url: youtube-nocookie.com/get_video_info?video_id=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
      - raw_url: www.youtube-nocookie.com/get_video_info?video_id=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/get_video_info?video_id=123ah
      - raw_url: www.youtube-nocookie.com/get_video_info?foo=bar
        unchanged: true # no video_id parameter
      - raw_url: www.youtubeqnocookie.com/get_video_info?video_id=123ah
        unchanged: true # improper hostname
  - name: youtube_thumbnails
    pattern: i\.ytimg\.com\/vi\/(.*?)\/.*?\.(\w*?)(?:\?.*|$)
    replace : i.ytimg.com.fuzzy.replayweb.page/vi/\1/thumbnail.\2
    tests:
      - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.jpg?sqp=-oaymwEmCIAKENAF8quKqQMa8AEB-AH-CYAC0AWKAgwIABABGHIgTyg-MA8=&rs=AOn4CLDr-FmDmP3aCsD84l48ygBmkwHg-g
        fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg
      - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.png?sqp=-oaymwEmCIAKENAF8quKqQMa8AEB-AH-CYAC0AWKAgwIABABGHIgTyg-MA8=&rs=AOn4CLDr-FmDmP3aCsD84l48ygBmkwHg-g
        fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.png
      - raw_url: i.ytimg.com/vi/-KpLmsAR23I/maxresdefault.jpg
        fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg
      - raw_url: i.ytimg.com/vi/-KpLmsAR23I/max-res.default.jpg
        fuzzified_url: i.ytimg.com.fuzzy.replayweb.page/vi/-KpLmsAR23I/thumbnail.jpg
  - name: trim_digits_only
    pattern: ([^?]+)\?[\d]+$
    replace : \1
    tests:
      - raw_url: www.example.com/page?1234
        fuzzified_url: www.example.com/page
      - raw_url: www.example.com/page?foo=1234
        unchanged: true
      - raw_url: www.example.com/page1234
        unchanged: true
      - raw_url: www.example.com/page?foo=bar&1234
        unchanged: true
      - raw_url: www.example.com/page?1234=bar
        unchanged: true
      - raw_url: www.example.com/page?1234&foo=bar
        unchanged: true
  - name: youtubei
    pattern: (?:www\.)?youtube(?:-nocookie)?\.com\/(youtubei\/[^?]+).*(videoId[^&]+).*
    replace : youtube.fuzzy.replayweb.page/\1?\2
    tests:
      - raw_url: www.youtube-nocookie.com/youtubei/page/?videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: youtube-nocookie.com/youtubei/page/?videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: youtube.com/youtubei/page/?videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: www.youtube.com/youtubei/page/?videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: youtube.com/youtubei/page/videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: youtube.com/youtubei/page/videoIdqqq=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoIdqqq=123ah
      - raw_url: youtube.com/youtubei/page/videoId=123ah&foo=bar
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: youtube.com/youtubei/page/?foo=bar&videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/?videoId=123ah
      - raw_url: youtube.com/youtubei/page/foo=bar&videoId=123ah
        fuzzified_url: youtube.fuzzy.replayweb.page/youtubei/page/foo=bar&?videoId=123ah
      - raw_url: youtube.com/youtubei/?videoId=123ah
        unchanged: true
  - name: youtube_embed
    pattern: (?:www\.)?youtube(?:-nocookie)?\.com/embed/([^?]+).*
    replace : youtube.fuzzy.replayweb.page/embed/\1
    tests:
      - raw_url: www.youtube-nocookie.com/embed/foo
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo
      - raw_url: www.youtube-nocookie.com/embed/bar
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/bar
      - raw_url: www.youtube-nocookie.com/embed/foo/bar
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo/bar
      - raw_url: www.youtube.com/embed/foo
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo
      - raw_url: youtube.com/embed/foo
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo
      - raw_url: youtube-nocookie.com/embed/foo
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo
      - raw_url: youtube.com/embed/foo?bar=alice
        fuzzified_url: youtube.fuzzy.replayweb.page/embed/foo

  - name: vimeo_cdn_fix # custom warc2zim rule intended to fix Vimeo support
    pattern: .*(?:gcs-vimeo|vod|vod-progressive|vod-adaptive)\.akamaized\.net.*\/(.+?.mp4)\?.*range=(.*?)(?:&.*|$)
    replace : vimeo-cdn.fuzzy.replayweb.page/\1?range=\2
    tests:
      - raw_url: gcs-vimeo.akamaized.net/123.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod.akamaized.net/123.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod-progressive.akamaized.net/123.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod-adaptive.akamaized.net/123.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod.akamaized.net/123.mp4?foo=bar&range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod.akamaized.net/123.mp4?foo=bar&range=123-456&bar=foo
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod.akamaized.net/123.mp4?range=123-456&bar=foo
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: foovod.akamaized.net/123.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/123.mp4?range=123-456
      - raw_url: vod.akamaized.net/1/23.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4?range=123-456
      - raw_url: vod.akamaized.net/a/23.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4?range=123-456
      - raw_url: vod.akamaized.net/foo/bar/23.mp4?range=123-456
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4?range=123-456
      - raw_url: foo.akamaized.net/123.mp4?range=123-456
        unchanged: true
  - name: vimeo_cdn
    pattern: .*(?:gcs-vimeo|vod|vod-progressive)\.akamaized\.net.*?\/([\d/]+.mp4)$
    replace : vimeo-cdn.fuzzy.replayweb.page/\1
    tests:
      - raw_url: vod.akamaized.net/23.mp4
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23.mp4
      - raw_url: vod.akamaized.net/23/12332.mp4
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/23/12332.mp4
      - raw_url: https://vod-progressive.akamaized.net/exp=1635528595~acl=%2Fvimeo-prod-skyfire-std-us%2F01%2F4423%2F13%2F347119375%2F1398505169.mp4~hmac=27c31f1990aab5e5429f7f7db5b2dcbcf8d2f5c92184d53102da36920d33d53e/vimeo-prod-skyfire-std-us/01/4423/13/347119375/1398505169.mp4
        fuzzified_url: vimeo-cdn.fuzzy.replayweb.page/01/4423/13/347119375/1398505169.mp4
  - name: vimeo_player
    pattern: .*player.vimeo.com\/(video\/[\d]+)\?.*
    replace : vimeo.fuzzy.replayweb.page/\1
    tests:
      - raw_url: player.vimeo.com/video/1234?foo=bar
        fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234
      - raw_url: foo.player.vimeo.com/video/1234?foo=bar
        fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234
      - raw_url: player.vimeo.com/video/1234?foo
        fuzzified_url: vimeo.fuzzy.replayweb.page/video/1234
      - raw_url: player.vimeo.com/video/1/23?foo=bar
        unchanged: true
      - raw_url: player.vimeo.com/video/123a?foo=bar
        unchanged: true
      - raw_url: player.vimeo.com/video/?foo=bar
        unchanged: true
  - name: i_vimeo_cdn
    pattern: .*i\.vimeocdn\.com\/(.*)\?.*
    replace : i.vimeocdn.fuzzy.replayweb.page/\1
    tests:
      - raw_url: i.vimeocdn.com/image/1234?foo=bar
        fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/image/1234
      - raw_url: i.vimeocdn.com/something/a456?foo
        fuzzified_url: i.vimeocdn.fuzzy.replayweb.page/something/a456
  - name: cheatography_com
    pattern: cheatography\.com\/scripts\/(.*).js.*[?&](v=[^&]+).*
    replace : cheatography.com.fuzzy.replayweb.page/scripts/\1.js?\2
    tests:
      - raw_url: cheatography.com/scripts/useful.min.js?v=2&q=1719438924
        fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2
      - raw_url: cheatography.com/scripts/foo.js?v=2&q=1719438924
        fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/foo.js?v=2
      - raw_url: cheatography.com/scripts/useful.min.js?q=1719438924&v=2
        fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2
      - raw_url: cheatography.com/scripts/useful.min.js?q=1719438924&v=2&foo=bar
        fuzzified_url: cheatography.com.fuzzy.replayweb.page/scripts/useful.min.js?v=2
  - name: der_postillon_com
    pattern: blogger.googleusercontent.com\/img\/(.*\.jpg)=.*
    replace: blogger.googleusercontent.com.fuzzy.replayweb.page/img/\1.resized
    tests:
      - raw_url: blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjlN4LY6kFVwL8-rinDWp3kJp1TowOVD8vq8TP8nl3Lf1sI-hx0DE1GQA1jw7DT7XvK3FjghzJ17_1pvyXyDBAV0vtigJRnFCNfMxnndBnN3NYoXUvKQQsQ7JTGXOSajdo0mNQIv8wss_AxPBMrR4-Dd_EEacV7ZMS3m_IL2dz0WsbbKn7FD7ntsfOe0JUq/s600-rw/tickerzugtier2.jpg=w487-h220-p-k-no-nu
        fuzzified_url: blogger.googleusercontent.com.fuzzy.replayweb.page/img/b/R29vZ2xl/AVvXsEjlN4LY6kFVwL8-rinDWp3kJp1TowOVD8vq8TP8nl3Lf1sI-hx0DE1GQA1jw7DT7XvK3FjghzJ17_1pvyXyDBAV0vtigJRnFCNfMxnndBnN3NYoXUvKQQsQ7JTGXOSajdo0mNQIv8wss_AxPBMrR4-Dd_EEacV7ZMS3m_IL2dz0WsbbKn7FD7ntsfOe0JUq/s600-rw/tickerzugtier2.jpg.resized
      - raw_url: blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjlN4LY6kFVwL8-rinDWp3kJp1TowOVD8vq8TP8nl3Lf1sI-hx0DE1GQA1jw7DT7XvK3FjghzJ17_1pvyXyDBAV0vtigJRnFCNfMxnndBnN3NYoXUvKQQsQ7JTGXOSajdo0mNQIv8wss_AxPBMrR4-Dd_EEacV7ZMS3m_IL2dz0WsbbKn7FD7ntsfOe0JUq/w72-h72-p-k-no-nu/tickerzugtier2.jpg
        unchanged: true
  - name: iranwire_com
    pattern: (iranwire\.com\/questions\/detail\/.*)\?.*
    replace: \1
    tests:
      - raw_url: iranwire.com/questions/detail/1723?&_=1721804954220
        fuzzified_url: iranwire.com/questions/detail/1723
      - raw_url: iranwire.com/questions/detail/1725?foo=bar&_=1721804454220
        fuzzified_url: iranwire.com/questions/detail/1725
