☰

TS4 gallery downloader

Download households, lots and rooms from The Sims 4 Gallery website

  1. // ==UserScript==
  2. // @name        TS4 gallery downloader
  3. // @description Download households, lots and rooms from The Sims 4 Gallery website
  4. // @author      anadius
  5. // @match       *://www.ea.com/*/games/the-sims/the-sims-4/pc/gallery*
  6. // @match       *://www.ea.com/games/the-sims/the-sims-4/pc/gallery*
  7. // @connect     sims4cdn.ea.com
  8. // @connect     athena.thesims.com
  9. // @connect     www.thesims.com
  10. // @connect     thesims-api.ea.com
  11. // @version     2.2.0
  12. // @namespace   anadius.github.io
  13. // @grant       unsafeWindow
  14. // @grant       GM.xmlHttpRequest
  15. // @grant       GM_xmlhttpRequest
  16. // @grant       GM.getResourceUrl
  17. // @grant       GM_getResourceURL
  18. // @icon        https://anadius.github.io/ts4installer-tumblr-files/userjs/sims-4-gallery-downloader.png
  19. // @resource    bundle.json https://anadius.github.io/ts4installer-tumblr-files/userjs/bundle.min.json?version=1.113.291
  20. // @require     https://greasemonkey.github.io/gm4-polyfill/gm4-polyfill.js
  21. // @require     https://cdn.jsdelivr.net/npm/long@4.0.0/dist/long.js#sha256-Cp9yM71yBwlF4CLQBfDKHoxvI4BoZgQK5aKPAqiupEQ=
  22. // @require     https://cdn.jsdelivr.net/npm/file-saver@2.0.1/dist/FileSaver.min.js#sha256-Sf4Tr1mzejErqH+d3jzEfBiRJAVygvjfwUbgYn92yOU=
  23. // @require     https://cdn.jsdelivr.net/npm/jszip@3.2.0/dist/jszip.min.js#sha256-VwkT6wiZwXUbi2b4BOR1i5hw43XMzVsP88kpesvRYfU=
  24. // @require     https://cdn.jsdelivr.net/npm/protobufjs@6.8.8/dist/protobuf.min.js#sha256-VPK6lQo4BEjkmYz6rFWbuntzvMJmX45mSiLXgcLHCLE=
  25. // ==/UserScript==
  26.  
  27. /* global protobuf, saveAs, JSZip, Long */
  28. /* eslint curly: 0 */
  29. /* eslint no-sequences: 0 */
  30. /* eslint no-return-assign: 0 */
  31.  
  32. const TRAY_ITEM_URL = 'https://thesims-api.ea.com/api/gallery/v1/sims/{UUID}';
  33. const DATA_ITEM_URL = 'http://sims4cdn.ea.com/content-prod.ts4/prod/{FOLDER}/{GUID}.dat';
  34. const IMAGE_URL = 'https://athena.thesims.com/v2/images/{TYPE}/{FOLDER}/{GUID}/{INDEX}.jpg';
  35.  
  36. const EXCHANGE_HOUSEHOLD = 1;
  37. const EXCHANGE_BLUEPRINT = 2;
  38. const EXCHANGE_ROOM = 3;
  39. const EXCHANGE_TATOO = 5;
  40. const EXTENSIONS = {
  41.   [EXCHANGE_HOUSEHOLD]: ['Household', 'householdbinary', 'hhi', 'sgi'],
  42.   [EXCHANGE_BLUEPRINT]: ['Lot', 'blueprint', 'bpi', 'bpi'],
  43.   [EXCHANGE_ROOM]: ['Room', 'room', 'rmi', null],
  44.   [EXCHANGE_TATOO]: ['Tattoo', 'part', 'pti', 'pti'],
  45. };
  46. const IMAGE_TYPE = {
  47.   [EXCHANGE_HOUSEHOLD]: 0,
  48.   [EXCHANGE_BLUEPRINT]: 1,
  49.   [EXCHANGE_ROOM]: 2,
  50.   [EXCHANGE_TATOO]: 3,
  51. };
  52. const ADDITIONAL_IMAGE_GROUP = {
  53.   [EXCHANGE_HOUSEHOLD]: 0x10,
  54.   [EXCHANGE_BLUEPRINT]: 0x100,
  55.   [EXCHANGE_TATOO]: 0x10,
  56. };
  57.  
  58. const BIG_WIDTH = 591;
  59. const BIG_HEIGHT = 394;
  60. const SMALL_WIDTH = 300;
  61. const SMALL_HEIGHT = 200;
  62.  
  63. const LONG_TYPES = [
  64.   "sint64",
  65.   "uint64",
  66.   "int64",
  67.   "sfixed64",
  68.   "fixed64"
  69. ];
  70.  
  71. /* helper functions */
  72.  
  73. const getRandomIntInclusive = (min, max) => {
  74.   min = Math.ceil(min);
  75.   max = Math.floor(max);
  76.   return Math.floor(Math.random() * (max - min + 1)) + min;
  77. };
  78.  
  79. const reportError = e => {
  80.   if(e.name && e.message && e.stack)
  81.     alert(`${e.name}\n\n${e.message}\n\n${e.stack}`);
  82.   else
  83.     alert(e);
  84. };
  85.  
  86. const xhr = details => new Promise((resolve, reject) => {
  87.   const stack = new Error().stack;
  88.   const reject_xhr = res => {
  89.     console.log(res);
  90.     reject({
  91.       name: 'GMXHRError',
  92.       message: `XHR for URL ${details.url} returned status code ${res.status}`,
  93.       stack: stack,
  94.       status: res.status
  95.     });
  96.   };
  97.   GM.xmlHttpRequest(Object.assign(
  98.     {method: 'GET'},
  99.     details,
  100.     {
  101.       onload: res => {
  102.         if(res.status === 404)
  103.           reject_xhr(res);
  104.         else
  105.           resolve(res.response);
  106.       },
  107.       onerror: res => reject_xhr
  108.     }
  109.   ));
  110. });
  111.  
  112. /* functions taken from thesims.min.js */
  113.  
  114. function dashify(uuid) {
  115.   var slice = String.prototype.slice,
  116.       indices = [
  117.         [0, 8],
  118.         [8, 12],
  119.         [12, 16],
  120.         [16, 20],
  121.         [20]
  122.       ];
  123.   return indices.map(function(index) {
  124.     return slice.apply(uuid, index)
  125.   }).join("-")
  126. }
  127.  
  128. function uuid2Guid(uuid) {
  129.   if (-1 !== uuid.indexOf("-")) return uuid.toUpperCase();
  130.   var decoded;
  131.   try {
  132.     decoded = atob(uuid)
  133.   } catch (err) {
  134.     return !1
  135.   }
  136.   for (var guid = "", i = 0; i < decoded.length; i++) {
  137.     var ch = decoded.charCodeAt(i);
  138.     ch = (240 & ch) >> 4, ch = ch.toString(16);
  139.     var tmpstr = ch.toString();
  140.     ch = decoded.charCodeAt(i), ch = 15 & ch, ch = ch.toString(16), tmpstr += ch.toString(), guid += tmpstr
  141.   }
  142.   return dashify(guid).toUpperCase()
  143. }
  144.  
  145. function getFilePath(guid) {
  146.   var bfnvInit = 2166136261;
  147.   for (var fnvInit = bfnvInit, i = 0; i < guid.length; ++i) fnvInit += (fnvInit << 1) + (fnvInit << 4) + (fnvInit << 7) + (fnvInit << 8) + (fnvInit << 24), fnvInit ^= guid.charCodeAt(i);
  148.   var result = (fnvInit >>> 0) % 1e4;
  149.   return result = result.toString(16), result = "0x" + "00000000".substr(0, 8 - result.length) + result
  150. };
  151.  
  152. // wrap everything in async anonymous function
  153. (async () => {
  154.  
  155. /* tray item */
  156.  
  157. const getRandomId = () => {
  158.   return new Long(
  159.     getRandomIntInclusive(1, 0xffffffff),
  160.     getRandomIntInclusive(0, 0xffffffff),
  161.     true);
  162. };
  163.  
  164. const createPrefix = num => {
  165.   const arr = new ArrayBuffer(8);
  166.   const view = new DataView(arr);
  167.   view.setUint32(4, num, true);
  168.   return new Uint8Array(arr);
  169. };
  170.  
  171. const parseValue = (value, fieldType, isParentArray) => {
  172.   let _;
  173.   let parsedValue = value;
  174.   const valueType = typeof value;
  175.   if(valueType === "object") {
  176.     if(Array.isArray(value)) {
  177.       if(isParentArray) {
  178.         throw "No clue how to handle array of arrays"
  179.       }
  180.       parsedValue = parseMessageArray(value, fieldType);
  181.     }
  182.     else {
  183.       [parsedValue, _] = parseMessageObj(value, fieldType);
  184.     }
  185.   }
  186.   else if(valueType === "string") {
  187.     if(fieldType === "string" || fieldType === "bytes") {
  188.       // no processing needed
  189.     }
  190.     else if(LONG_TYPES.includes(fieldType)) {
  191.       parsedValue = Long.fromValue(value);
  192.     }
  193.     else {
  194.       // value is enum, lookup the enum and extract the actual value
  195.       parsedValue = root.lookupEnum(fieldType).values[value.split('.').pop()];
  196.     }
  197.   }
  198.  
  199.   return parsedValue;
  200. };
  201.  
  202. const parseMessageArray = (messageArray, className) => {
  203.   const parsedArray = [];
  204.   messageArray.forEach(arrayItem => {
  205.     parsedArray.push(parseValue(arrayItem, className, true));
  206.   });
  207.   return parsedArray;
  208. };
  209.  
  210. const camelToSnakeCase = str => str.replace(/[A-Z]/g, letter => `_${letter.toLowerCase()}`);
  211.  
  212. // EA likes to mess things up, in proto files they use snake case, in JSON they use camel case
  213. // but there are some exceptions, so try both
  214. const findKeyName = (key, messageClass) => {
  215.   const fields = messageClass.fields;
  216.   if(typeof messageClass.fields[key] !== "undefined")
  217.     return key;
  218.  
  219.   const key2 = camelToSnakeCase(key);
  220.   if(typeof messageClass.fields[key2] !== "undefined")
  221.     return key2;
  222.  
  223.   const nameParts = [];
  224.   let msgClass = messageClass;
  225.   while(msgClass.name !== "" && typeof msgClass.name !== "undefined") {
  226.     nameParts.unshift(msgClass.name);
  227.     msgClass = msgClass.parent;
  228.   }
  229.   const name = nameParts.join(".");
  230.   throw `${name} class doesn't have ${key} nor ${key2} key.`
  231. };
  232.  
  233. const parseMessageObj = (messageObj, className) => {
  234.   const keys = Object.keys(messageObj);
  235.  
  236.   const messageClass = root.lookupType(className);
  237.   const parsedMessage = {};
  238.   for(let i=0, l=keys.length; i<l; ++i) {
  239.     // this makes sure that `key` is in `messageClass.fields`
  240.     const key = findKeyName(keys[i], messageClass);
  241.     parsedMessage[key] = parseValue(messageObj[keys[i]], messageClass.fields[key].type);
  242.   }
  243.  
  244.   return [parsedMessage, messageClass];
  245. };
  246.  
  247. const getTrayItem = async (uuid, guid, folder) => {
  248.   let message;
  249.  
  250.   try {
  251.     message = await xhr({
  252.       url: TRAY_ITEM_URL.replace('{UUID}', encodeURIComponent(uuid)),
  253.       responseType: 'json',
  254.       headers: {
  255.         'Accept-Language': 'en-US,en;q=0.9',
  256.         'Cookie': ''
  257.       }
  258.     });
  259.   }
  260.   catch(e) {
  261.     if(e.name === 'GMXHRError') message = null;
  262.     else throw e;
  263.   }
  264.  
  265.   if(message === null || typeof message.error !== 'undefined')
  266.     throw "Can't download tray file. This item was most probably deleted.";
  267.  
  268.   const [parsedMessage, messageClass] = parseMessageObj(message, "EA.Sims4.Network.TrayMetadata");
  269.   // generate random ID
  270.   parsedMessage.id = getRandomId();
  271.  
  272.   // get number of additional images; set sim IDs
  273.   let additional = 0;
  274.   if(parsedMessage.type === EXCHANGE_BLUEPRINT)
  275.     additional = parsedMessage.metadata.bp_metadata.num_thumbnails - 1;
  276.   else if(parsedMessage.type === EXCHANGE_HOUSEHOLD) {
  277.     additional = parsedMessage.metadata.hh_metadata.sim_data.length;
  278.     // the same logic of making new IDs is used later to fix the main data file
  279.     parsedMessage.metadata.hh_metadata.sim_data.forEach((sim, i) => {
  280.       sim.id = parsedMessage.id.add(i + 1);
  281.     });
  282.   }
  283.  
  284.   const encodedMessage = messageClass.encode(parsedMessage).finish();
  285.   const prefix = createPrefix(encodedMessage.byteLength);
  286.   const resultFile = new Uint8Array(prefix.length + encodedMessage.length);
  287.   resultFile.set(prefix);
  288.   resultFile.set(encodedMessage, prefix.length);
  289.  
  290.   return [
  291.     resultFile, parsedMessage.type, parsedMessage.id, additional,
  292.     parsedMessage.modifier_name || parsedMessage.creator_name, parsedMessage.name
  293.   ];
  294. };
  295.  
  296. /* data file */
  297. const getDataItem = async (guid, folder, type, id) => {
  298.   let response;
  299.   try {
  300.     response = await xhr({
  301.       url: DATA_ITEM_URL.replace('{FOLDER}', folder).replace('{GUID}', guid),
  302.       responseType: 'arraybuffer'
  303.     });
  304.   }
  305.   catch(e) {
  306.     if(e.name === 'GMXHRError' && e.status === 404)
  307.       throw "Can't download data file. This item was most probably deleted.";
  308.     else
  309.       throw e;
  310.   }
  311.   if(type === EXCHANGE_HOUSEHOLD) {
  312.     const messageClass = root.lookupTypeOrEnum('EA.Sims4.Network.FamilyData');
  313.     let messageOffset, dataSuffixOffset;
  314.  
  315.     const view = new DataView(response);
  316.     const dataVersion = view.getUint32(0, true); // read the format version
  317.  
  318.     if(dataVersion < 2) {
  319.       messageOffset = 8;
  320.       dataSuffixOffset = response.byteLength;
  321.     }
  322.     else {
  323.       const mainItemLength = view.getUint32(4, true);
  324.       messageOffset = 16;
  325.       dataSuffixOffset = 4 + mainItemLength;
  326.     }
  327.  
  328.     const messageLength = view.getUint32(messageOffset - 4, true); // read message length
  329.     const message = messageClass.decode(new Uint8Array(response, messageOffset, messageLength)); // read and decode message
  330.     // read the suffixes
  331.     const messageSuffixOffset = messageOffset + messageLength;
  332.     const messageSuffix = new Uint8Array(response, messageSuffixOffset, dataSuffixOffset - messageSuffixOffset);
  333.     const dataSuffix = new Uint8Array(response, dataSuffixOffset);
  334.  
  335.     const newIdsDict = {};
  336.     const sims = message.family_account.sim;
  337.     sims.forEach((sim, i) => {
  338.       newIdsDict[sim.sim_id.toString()] = id.add(1+i);
  339.     });
  340.     sims.forEach(sim => {
  341.       sim.sim_id = newIdsDict[sim.sim_id.toString()];
  342.       sim.significant_other = newIdsDict[sim.significant_other.toString()];
  343.       sim.attributes.genealogy_tracker.family_relations.forEach(relation => {
  344.         const newId = newIdsDict[relation.sim_id.toString()];
  345.         if(typeof newId !== "undefined") {
  346.           relation.sim_id = newId;
  347.         }
  348.       });
  349.     });
  350.  
  351.     try {
  352.       const editedMessage = new Uint8Array(messageClass.encode(message).finish());
  353.       const resultArray = new Uint8Array(messageOffset + editedMessage.length + messageSuffix.length + dataSuffix.length);
  354.       const resultView = new DataView(resultArray.buffer);
  355.       resultView.setUint32(0, dataVersion, true);
  356.       if(dataVersion >= 2) {
  357.         resultView.setUint32(4, 8 + editedMessage.length + messageSuffix.length, true);
  358.       }
  359.       resultView.setUint32(messageOffset - 4, editedMessage.length, true);
  360.       resultArray.set(editedMessage, messageOffset);
  361.       resultArray.set(messageSuffix, messageOffset + editedMessage.length);
  362.       resultArray.set(dataSuffix, messageOffset + editedMessage.length + messageSuffix.length);
  363.       return resultArray.buffer;
  364.     }
  365.     catch(err) {
  366.       console.error(err);
  367.       return response;
  368.     }
  369.   }
  370.   else
  371.     return response;
  372. };
  373.  
  374. /* image files */
  375.  
  376. const loadImage = url => new Promise(resolve => {
  377.   xhr({
  378.     url: url,
  379.     responseType: 'blob'
  380.   }).then(response => {
  381.     const urlCreator = window.URL || window.webkitURL;
  382.     const imageUrl = urlCreator.createObjectURL(response);
  383.  
  384.     const img = new Image();
  385.     img.onload = () => {
  386.       urlCreator.revokeObjectURL(img.src);
  387.       resolve(img);
  388.     };
  389.     img.src = imageUrl;
  390.   });
  391. });
  392.  
  393. const newCanvas = (width, height) => {
  394.   const canvas = document.createElement('canvas');
  395.   canvas.width = width;
  396.   canvas.height = height;
  397.   return canvas;
  398. };
  399.  
  400. const getImages = async (guid, folder, type, additional) => {
  401.   const URL_TEMPLATE = IMAGE_URL.replace('{FOLDER}', folder).replace('{GUID}', guid).replace('{TYPE}', IMAGE_TYPE[type]);
  402.   const big = newCanvas(BIG_WIDTH, BIG_HEIGHT);
  403.   const small = newCanvas(SMALL_WIDTH, SMALL_HEIGHT);
  404.   const images = [];
  405.   for(let i=0; i<=additional; ++i) {
  406.     let url = URL_TEMPLATE.replace('{INDEX}', i.toString().padStart(2, '0'));
  407.     let img = await loadImage(url);
  408.     let x, y, width, height;
  409.  
  410.     if(type == EXCHANGE_BLUEPRINT || (type == EXCHANGE_HOUSEHOLD && i > 0)) {
  411.       width = Math.round(img.naturalHeight * BIG_WIDTH / BIG_HEIGHT);
  412.       height = img.naturalHeight;
  413.     }
  414.     else {
  415.       width = BIG_WIDTH;
  416.       height = BIG_HEIGHT;
  417.     }
  418.     x = (img.naturalWidth - width) / 2;
  419.     y = (img.naturalHeight - height) / 2;
  420.  
  421.     if(i == 0) {
  422.       small.getContext('2d').drawImage(img, x, y, width, height, 0, 0, SMALL_WIDTH, SMALL_HEIGHT);
  423.       images.push(small.toDataURL('image/jpeg').split('base64,')[1]);
  424.     }
  425.     big.getContext('2d').drawImage(img, x, y, width, height, 0, 0, BIG_WIDTH, BIG_HEIGHT);
  426.     images.push(big.toDataURL('image/jpeg').split('base64,')[1]);
  427.   }
  428.   return images;
  429. };
  430.  
  431. /* main download */
  432.  
  433. const generateName = (type, id, ext) => {
  434.   const typeStr = '0x' + type.toString(16).toLowerCase().padStart(8, 0);
  435.   const idStr = '0x' + id.toString(16).toLowerCase().padStart(16, 0);
  436.   return typeStr + '!' + idStr + '.' + ext;
  437. };
  438.  
  439. const toggleDownload = (scope, downloading) => {
  440.   scope.vm.toggleDownload.toggling = downloading;
  441.   scope.$apply();
  442. };
  443.  
  444. const downloadItem = async scope => {
  445.   try {
  446.     const uuid = scope.vm.uuid;
  447.     const guid = uuid2Guid(uuid);
  448.     const folder = getFilePath(guid);
  449.  
  450.     toggleDownload(scope, true);
  451.     const zip = new JSZip();
  452.  
  453.     const [trayItem, type, id, additional, author, title] = await getTrayItem(uuid, guid, folder);
  454.     zip.file(generateName(type, id, 'trayitem'), trayItem);
  455.  
  456.     const [typeStr, dataExt, imageExt, additionalExt] = EXTENSIONS[type];
  457.  
  458.     const dataItem = await getDataItem(guid, folder, type, id);
  459.     zip.file(generateName(0, id, dataExt), dataItem);
  460.  
  461.     const images = await getImages(guid, folder, type, additional);
  462.     images.forEach((data, i) => {
  463.       let group = i == 0 ? 2 : 3;
  464.       let extension = i < 2 ? imageExt : additionalExt;
  465.       let newId = id;
  466.       if(i >= 2) {
  467.         let j = i - 1;
  468.         group += ADDITIONAL_IMAGE_GROUP[type] * j;
  469.         if(type == EXCHANGE_HOUSEHOLD)
  470.           newId = newId.add(j);
  471.       }
  472.       zip.file(generateName(group, newId, extension), data, {base64: true});
  473.     });
  474.  
  475.     let filename = [author, typeStr, title, uuid.replace(/\+/g, '-').replace(/\//g, '_')].join('__');
  476.     filename = filename.replace(/\s+/g, '_').replace(/[^a-z0-9\.\-=_]/gi, '');
  477.     const content = await zip.generateAsync({type:'blob'});
  478.     saveAs(content, filename + '.zip');
  479.   }
  480.   catch(e) {
  481.     reportError(e);
  482.   }
  483.   toggleDownload(scope, false);
  484. };
  485.  
  486. /* init */
  487.  
  488. let data = await fetch(await GM.getResourceUrl('bundle.json'));
  489. let jsonDescriptor = await data.json();
  490. const root = protobuf.Root.fromJSON(jsonDescriptor);
  491.  
  492. document.addEventListener('click', e => {
  493.   let el = e.target;
  494.   if(el.tagName === 'SPAN')
  495.     el = el.parentNode.parentNode;
  496.   else if(el.tagName === 'A')
  497.     el = el.parentNode;
  498.  
  499.   if(el.tagName === 'LI' && el.classList.contains('stream-tile__actions-download')) {
  500.     e.stopPropagation();
  501.     const scope = unsafeWindow.angular.element(el).scope();
  502.     downloadItem(scope);
  503.   }
  504. }, true);
  505.  
  506. console.log('running');
  507.  
  508. })();
  509.  
  510. /* add "force login" link */
  511.  
  512. const a = document.createElement('a');
  513. a.href = 'https://www.thesims.com/login?redirectUri=' + encodeURIComponent(document.location);
  514. a.innerHTML = '<b>force login</b>';
  515. a.style.background = 'grey';
  516. a.style.color = 'white';
  517. a.style.display = 'inline-block';
  518. a.style.position = 'absolute';
  519. a.style.top = 0;
  520. a.style.left = 0;
  521. a.style.height = '40px';
  522. a.style.lineHeight = '40px';
  523. a.style.padding = '0 15px';
  524. a.style.zIndex = 99999;
  525. document.body.appendChild(a);