You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

136 lines
5.1 KiB

  1. const inflate = require('tiny-inflate');
  2. const { swap32LE } = require('./swap');
  3. // Shift size for getting the index-1 table offset.
  4. const SHIFT_1 = 6 + 5;
  5. // Shift size for getting the index-2 table offset.
  6. const SHIFT_2 = 5;
  7. // Difference between the two shift sizes,
  8. // for getting an index-1 offset from an index-2 offset. 6=11-5
  9. const SHIFT_1_2 = SHIFT_1 - SHIFT_2;
  10. // Number of index-1 entries for the BMP. 32=0x20
  11. // This part of the index-1 table is omitted from the serialized form.
  12. const OMITTED_BMP_INDEX_1_LENGTH = 0x10000 >> SHIFT_1;
  13. // Number of entries in an index-2 block. 64=0x40
  14. const INDEX_2_BLOCK_LENGTH = 1 << SHIFT_1_2;
  15. // Mask for getting the lower bits for the in-index-2-block offset. */
  16. const INDEX_2_MASK = INDEX_2_BLOCK_LENGTH - 1;
  17. // Shift size for shifting left the index array values.
  18. // Increases possible data size with 16-bit index values at the cost
  19. // of compactability.
  20. // This requires data blocks to be aligned by DATA_GRANULARITY.
  21. const INDEX_SHIFT = 2;
  22. // Number of entries in a data block. 32=0x20
  23. const DATA_BLOCK_LENGTH = 1 << SHIFT_2;
  24. // Mask for getting the lower bits for the in-data-block offset.
  25. const DATA_MASK = DATA_BLOCK_LENGTH - 1;
  26. // The part of the index-2 table for U+D800..U+DBFF stores values for
  27. // lead surrogate code _units_ not code _points_.
  28. // Values for lead surrogate code _points_ are indexed with this portion of the table.
  29. // Length=32=0x20=0x400>>SHIFT_2. (There are 1024=0x400 lead surrogates.)
  30. const LSCP_INDEX_2_OFFSET = 0x10000 >> SHIFT_2;
  31. const LSCP_INDEX_2_LENGTH = 0x400 >> SHIFT_2;
  32. // Count the lengths of both BMP pieces. 2080=0x820
  33. const INDEX_2_BMP_LENGTH = LSCP_INDEX_2_OFFSET + LSCP_INDEX_2_LENGTH;
  34. // The 2-byte UTF-8 version of the index-2 table follows at offset 2080=0x820.
  35. // Length 32=0x20 for lead bytes C0..DF, regardless of SHIFT_2.
  36. const UTF8_2B_INDEX_2_OFFSET = INDEX_2_BMP_LENGTH;
  37. const UTF8_2B_INDEX_2_LENGTH = 0x800 >> 6; // U+0800 is the first code point after 2-byte UTF-8
  38. // The index-1 table, only used for supplementary code points, at offset 2112=0x840.
  39. // Variable length, for code points up to highStart, where the last single-value range starts.
  40. // Maximum length 512=0x200=0x100000>>SHIFT_1.
  41. // (For 0x100000 supplementary code points U+10000..U+10ffff.)
  42. //
  43. // The part of the index-2 table for supplementary code points starts
  44. // after this index-1 table.
  45. //
  46. // Both the index-1 table and the following part of the index-2 table
  47. // are omitted completely if there is only BMP data.
  48. const INDEX_1_OFFSET = UTF8_2B_INDEX_2_OFFSET + UTF8_2B_INDEX_2_LENGTH;
  49. // The alignment size of a data block. Also the granularity for compaction.
  50. const DATA_GRANULARITY = 1 << INDEX_SHIFT;
  51. class UnicodeTrie {
  52. constructor(data) {
  53. const isBuffer = (typeof data.readUInt32BE === 'function') && (typeof data.slice === 'function');
  54. if (isBuffer || data instanceof Uint8Array) {
  55. // read binary format
  56. let uncompressedLength;
  57. if (isBuffer) {
  58. this.highStart = data.readUInt32LE(0);
  59. this.errorValue = data.readUInt32LE(4);
  60. uncompressedLength = data.readUInt32LE(8);
  61. data = data.slice(12);
  62. } else {
  63. const view = new DataView(data.buffer);
  64. this.highStart = view.getUint32(0, true);
  65. this.errorValue = view.getUint32(4, true);
  66. uncompressedLength = view.getUint32(8, true);
  67. data = data.subarray(12);
  68. }
  69. // double inflate the actual trie data
  70. data = inflate(data, new Uint8Array(uncompressedLength));
  71. data = inflate(data, new Uint8Array(uncompressedLength));
  72. // swap bytes from little-endian
  73. swap32LE(data);
  74. this.data = new Uint32Array(data.buffer);
  75. } else {
  76. // pre-parsed data
  77. ({ data: this.data, highStart: this.highStart, errorValue: this.errorValue } = data);
  78. }
  79. }
  80. get(codePoint) {
  81. let index;
  82. if ((codePoint < 0) || (codePoint > 0x10ffff)) {
  83. return this.errorValue;
  84. }
  85. if ((codePoint < 0xd800) || ((codePoint > 0xdbff) && (codePoint <= 0xffff))) {
  86. // Ordinary BMP code point, excluding leading surrogates.
  87. // BMP uses a single level lookup. BMP index starts at offset 0 in the index.
  88. // data is stored in the index array itself.
  89. index = (this.data[codePoint >> SHIFT_2] << INDEX_SHIFT) + (codePoint & DATA_MASK);
  90. return this.data[index];
  91. }
  92. if (codePoint <= 0xffff) {
  93. // Lead Surrogate Code Point. A Separate index section is stored for
  94. // lead surrogate code units and code points.
  95. // The main index has the code unit data.
  96. // For this function, we need the code point data.
  97. index = (this.data[LSCP_INDEX_2_OFFSET + ((codePoint - 0xd800) >> SHIFT_2)] << INDEX_SHIFT) + (codePoint & DATA_MASK);
  98. return this.data[index];
  99. }
  100. if (codePoint < this.highStart) {
  101. // Supplemental code point, use two-level lookup.
  102. index = this.data[(INDEX_1_OFFSET - OMITTED_BMP_INDEX_1_LENGTH) + (codePoint >> SHIFT_1)];
  103. index = this.data[index + ((codePoint >> SHIFT_2) & INDEX_2_MASK)];
  104. index = (index << INDEX_SHIFT) + (codePoint & DATA_MASK);
  105. return this.data[index];
  106. }
  107. return this.data[this.data.length - DATA_GRANULARITY];
  108. }
  109. }
  110. module.exports = UnicodeTrie;