Replace emoji from a LUT

.github/dependabot.yaml

@@ -1,11 +0,0 @@
----
-version: 2
-updates:
-  - package-ecosystem: 'bundler'
-    directory: '/'
-    schedule:
-        interval: 'weekly'
-  - package-ecosystem: 'github-actions'
-    directory: '/'
-    schedule:
-        interval: 'weekly'

.github/workflows/test.yml

@@ -1,24 +0,0 @@
-name: Test
-
-on: [push, pull_request]
-
-jobs:
-  test:
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby: ["2.7", "3.0", "3.1"]
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v3
-    
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: ${{matrix.ruby}}
-        bundler-cache: true
-    
-    - name: Run tests
-      run: bundle exec rake

.gitignore

@@ -1,6 +1,2 @@
-/bin/*
-.bundle
 .ruby-version
-Gemfile.lock
-vendor/*
-!vendor/unicode-emoji-test.txt
+db/NamesList.txt

.travis.yml

@@ -0,0 +1,7 @@
+script: script/test
+rvm:
+  - 1.9.3
+  - 2.1.2
+  - 2.2
+notifications:
+  email: false

CONTRIBUTING.md

@@ -1,4 +1,5 @@
-Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [project's open source license](LICENSE).
+Our emoji set is based off Apple's emoji character palette, plus some custom
+emoji such as :octocat: :shipit: :metal:.
 
 Some useful tools in development are:
 
@@ -11,11 +12,28 @@ Sets up the development environment. The prerequisites are:
 * Ruby 1.9+
 * Bundler
 
+```
+rake db:generate
+```
+
+On OS X, this will rebuild the `db/Category-Emoji.json` file from the system
+one, pulling in any new emoji that Apple may have added in the meantime.
+
 ```
 script/test
 ```
 
-Runs the test suite.
+Runs the test suite, including the integrity test where we assert that we have
+covered each of Apple's emoji.
+
+```
+script/regenerate
+```
+
+Rebuilds the `db/emoji.json` file which is our main list of emoji: their
+canonical representations, descriptions, aliases, and tags. This requires OS X
+because Safari is used in the process to verify which character render as emoji
+and which render as ordinary Unicode glyphs from the current font.
 
 ```
 script/console

Gemfile

@@ -1,7 +1,7 @@
 source "https://rubygems.org"
 
-gem "rake", "~> 10.3.2"
-gem "minitest", "~> 5.3.5"
-gem "i18n", "~> 1.8.5"
+gem "ttfunk", "~> 1.4.0"
+gem "rake"
+gem "minitest"
 
 gemspec

Gemfile.lock

@@ -1,25 +1,20 @@
 PATH
   remote: .
   specs:
-    gemoji (4.1.0)
+    gemoji (2.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    concurrent-ruby (1.1.7)
-    i18n (1.8.5)
-      concurrent-ruby (~> 1.0)
     minitest (5.3.5)
     rake (10.3.2)
+    ttfunk (1.4.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   gemoji!
-  i18n (~> 1.8.5)
-  minitest (~> 5.3.5)
-  rake (~> 10.3.2)
-
-BUNDLED WITH
-   2.4.10
+  minitest
+  rake
+  ttfunk (~> 1.4.0)

LICENSE

@@ -1,4 +1,21 @@
-Copyright (c) 2019 GitHub, Inc.
+octocat, squirrel, shipit
+Copyright (c) 2013 GitHub Inc. All rights reserved.
+
+bowtie, neckbeard, fu
+Copyright (c) 2013 37signals, LLC. All rights reserved.
+
+feelsgood, finnadie, goberserk, godmode, hurtrealbad, rage 1-4, suspect
+Copyright (c) 2013 id Software. All rights reserved.
+
+trollface
+Copyright (c) 2013 whynne@deviantart. All rights reserved.
+
+All other images
+Copyright (c) 2013 Apple Inc. All rights reserved.
+
+Source code:
+
+Copyright (c) 2013 GitHub, Inc.
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

README.md

@@ -1,7 +1,7 @@
 gemoji
 ======
 
-This library contains character information about native emojis.
+Emoji images and names. See the LICENSE for copyright information.
 
 
 Installation
@@ -13,6 +13,37 @@ Add `gemoji` to your Gemfile.
 gem 'gemoji'
 ```
 
+**Sync images**
+
+Images can be copied to your public directory with `rake emoji` in your app. This is the recommended approach since the images will be available at a consistent location. This works best with cached formatted user content generated by tools like [html-pipeline](https://github.com/jch/html-pipeline).
+
+``` ruby
+# Rakefile
+load 'tasks/emoji.rake'
+```
+
+```
+$ rake emoji
+```
+
+**Assets Precompiling**
+
+If you must, you can manually add all the images to your asset load path.
+
+``` ruby
+# config/application.rb
+config.assets.paths << Emoji.images_path
+```
+
+Then have them compiled to public on deploy.
+
+``` ruby
+# config/application.rb
+config.assets.precompile << "emoji/**/*.png"
+```
+
+**WARNING** Since there are a ton of images, just adding the path may slow down other lookups if you aren't using it. Compiling all the emojis on deploy will add overhead to your deploy if even the images haven't changed. Theres just so many more superfluous files to iterate over. Also, the urls will be fingerprinted which may not be ideal for referencing from cached content.
+
 
 Example Rails Helper
 --------------------
@@ -26,7 +57,7 @@ module EmojiHelper
   def emojify(content)
     h(content).to_str.gsub(/:([\w+-]+):/) do |match|
       if emoji = Emoji.find_by_alias($1)
-        %(<img alt="#$1" src="#{image_path("emoji/#{emoji.image_filename}")}" style="vertical-align:middle" width="20" height="20" />)
+        %(<img alt="#$1" src="#{asset_path("emoji/#{emoji.image_filename}")}" style="vertical-align:middle" width="20" height="20" />)
       else
         match
       end
@@ -76,14 +107,6 @@ emoji.image_filename #=> "music.png"
 As you create new emoji, you must ensure that you also create and put the images
 they reference by their `image_filename` to your assets directory.
 
-You can customize `image_filename` with:
-
-```ruby
-emoji = Emoji.create("music") do |char|
-  char.image_filename = "subdirectory/my_emoji.gif"
-end
-```
-
 For existing emojis, you can edit the list of aliases or add new tags in an edit block:
 
 ```ruby

Rakefile

@@ -1,4 +1,5 @@
 require 'rake/testtask'
+require 'rake/extensiontask'
 
 task :default => :test
 
@@ -9,9 +10,7 @@ end
 
 namespace :db do
   desc %(Generate Emoji data files needed for development)
-  task :generate => [
-    'vendor/unicode-emoji-test.txt',
-  ]
+  task :generate => ['db/Category-Emoji.json', 'db/NamesList.txt']
 
   desc %(Dump a list of supported Emoji with Unicode descriptions and aliases)
   task :dump => :generate do
@@ -19,6 +18,35 @@ namespace :db do
   end
 end
 
-file 'vendor/unicode-emoji-test.txt' do |t|
-  system 'curl', '-fsSL', 'http://unicode.org/Public/emoji/15.0/emoji-test.txt', '-o', t.name
+emoji_plist = '/System/Library/Input Methods/CharacterPalette.app/Contents/Resources/Category-Emoji.plist'
+nameslist_url = 'http://www.unicode.org/Public/6.3.0/ucd/NamesList.txt'
+
+task 'db/Category-Emoji.json' do |t|
+  system "plutil -convert json -r '#{emoji_plist}' -o '#{t.name}'"
 end
+
+file 'db/NamesList.txt' do |t|
+  system "curl -fsSL '#{nameslist_url}' -o '#{t.name}'"
+end
+
+namespace :images do
+  desc %(Extract PNG images from Apple's "Apple Color Emoji.ttf" font)
+  task :extract do
+    require 'emoji/extractor'
+    gem_dir = File.dirname(File.realpath(__FILE__))
+    Emoji::Extractor.new(64, "#{gem_dir}/images/emoji/unicode").extract!
+  end
+end
+
+namespace :c do
+  task :headers do
+    require 'emoji/tables'
+    gem_dir = File.dirname(File.realpath(__FILE__))
+
+    File.open(File.join(gem_dir, "ext/gemoji/emoji.h"), "w") do |file|
+      file.puts(Emoji::Tables.generate_length_tables)
+    end
+  end
+end
+
+Rake::ExtensionTask.new('gemoji')

db/Category-Emoji.json

@@ -0,0 +1,47 @@
+{
+  "EmojiDataArray" : [
+    {
+      "CVDataTitle" : "EmojiCategory-People",
+      "CVCategoryImage" : "Emoji-HumanImage",
+      "CVCategoryData" : {
+        "CVSkipNullGlyphs" : true,
+        "Data" : "😄,😃,😀,😊,☺️,😉,😍,😘,😚,😗,😙,😜,😝,😛,😳,😁,😔,😌,😒,😞,😣,😢,😂,😭,😪,😥,😰,😅,😓,😩,😫,😨,😱,😠,😡,😤,😖,😆,😋,😷,😎,😴,😵,😲,😟,😦,😧,😈,👿,😮,😬,😐,😕,😯,😶,😇,😏,😑,👲,👳,👮,👷,💂,👶,👦,👧,👨,👩,👴,👵,👱,👼,👸,😺,😸,😻,😽,😼,🙀,😿,😹,😾,👹,👺,🙈,🙉,🙊,💀,👽,💩,🔥,✨,🌟,💫,💥,💢,💦,💧,💤,💨,👂,👀,👃,👅,👄,👍,👎,👌,👊,✊,✌️,👋,✋,👐,👆,👇,👉,👈,🙌,🙏,☝️,👏,💪,🚶,🏃,💃,👫,👪,👬,👭,💏,💑,👯,🙆,🙅,💁,🙋,💆,💇,💅,👰,🙎,🙍,🙇,🎩,👑,👒,👟,👞,👡,👠,👢,👕,👔,👚,👗,🎽,👖,👘,👙,💼,👜,👝,👛,👓,🎀,🌂,💄,💛,💙,💜,💚,❤️,💔,💗,💓,💕,💖,💞,💘,💌,💋,💍,💎,👤,👥,💬,👣,💭"
+      }
+    },
+    {
+      "CVDataTitle" : "EmojiCategory-Nature",
+      "CVCategoryImage" : "Emoji-NatureImage",
+      "CVCategoryData" : {
+        "CVSkipNullGlyphs" : true,
+        "Data" : "🐶,🐺,🐱,🐭,🐹,🐰,🐸,🐯,🐨,🐻,🐷,🐽,🐮,🐗,🐵,🐒,🐴,🐑,🐘,🐼,🐧,🐦,🐤,🐥,🐣,🐔,🐍,🐢,🐛,🐝,🐜,🐞,🐌,🐙,🐚,🐠,🐟,🐬,🐳,🐋,🐄,🐏,🐀,🐃,🐅,🐇,🐉,🐎,🐐,🐓,🐕,🐖,🐁,🐂,🐲,🐡,🐊,🐫,🐪,🐆,🐈,🐩,🐾,💐,🌸,🌷,🍀,🌹,🌻,🌺,🍁,🍃,🍂,🌿,🌾,🍄,🌵,🌴,🌲,🌳,🌰,🌱,🌼,🌐,🌞,🌝,🌚,🌑,🌒,🌓,🌔,🌕,🌖,🌗,🌘,🌜,🌛,🌙,🌍,🌎,🌏,🌋,🌌,🌠,⭐️,☀️,⛅️,☁️,⚡️,☔️,❄️,⛄️,🌀,🌁,🌈,🌊"
+      }
+    },
+    {
+      "CVDataTitle" : "EmojiCategory-Objects",
+      "CVCategoryImage" : "Emoji-ObjectsImage",
+      "CVCategoryData" : {
+        "CVSkipNullGlyphs" : true,
+        "Data" : "🎍,💝,🎎,🎒,🎓,🎏,🎆,🎇,🎐,🎑,🎃,👻,🎅,🎄,🎁,🎋,🎉,🎊,🎈,🎌,🔮,🎥,📷,📹,📼,💿,📀,💽,💾,💻,📱,☎️,📞,📟,📠,📡,📺,📻,🔊,🔉,🔈,🔇,🔔,🔕,📢,📣,⏳,⌛️,⏰,⌚️,🔓,🔒,🔏,🔐,🔑,🔎,💡,🔦,🔆,🔅,🔌,🔋,🔍,🛁,🛀,🚿,🚽,🔧,🔩,🔨,🚪,🚬,💣,🔫,🔪,💊,💉,💰,💴,💵,💷,💶,💳,💸,📲,📧,📥,📤,✉️,📩,📨,📯,📫,📪,📬,📭,📮,📦,📝,📄,📃,📑,📊,📈,📉,📜,📋,📅,📆,📇,📁,📂,✂️,📌,📎,✒️,✏️,📏,📐,📕,📗,📘,📙,📓,📔,📒,📚,📖,🔖,📛,🔬,🔭,📰,🎨,🎬,🎤,🎧,🎼,🎵,🎶,🎹,🎻,🎺,🎷,🎸,👾,🎮,🃏,🎴,🀄️,🎲,🎯,🏈,🏀,⚽️,⚾️,🎾,🎱,🏉,🎳,⛳️,🚵,🚴,🏁,🏇,🏆,🎿,🏂,🏊,🏄,🎣,☕️,🍵,🍶,🍼,🍺,🍻,🍸,🍹,🍷,🍴,🍕,🍔,🍟,🍗,🍖,🍝,🍛,🍤,🍱,🍣,🍥,🍙,🍘,🍚,🍜,🍲,🍢,🍡,🍳,🍞,🍩,🍮,🍦,🍨,🍧,🎂,🍰,🍪,🍫,🍬,🍭,🍯,🍎,🍏,🍊,🍋,🍒,🍇,🍉,🍓,🍑,🍈,🍌,🍐,🍍,🍠,🍆,🍅,🌽"
+      }
+    },
+    {
+      "CVDataTitle" : "EmojiCategory-Places",
+      "CVCategoryImage" : "Emoji-PlacesImage",
+      "CVCategoryData" : {
+        "CVSkipNullGlyphs" : true,
+        "Data" : "🏠,🏡,🏫,🏢,🏣,🏥,🏦,🏪,🏩,🏨,💒,⛪️,🏬,🏤,🌇,🌆,🏯,🏰,⛺️,🏭,🗼,🗾,🗻,🌄,🌅,🌃,🗽,🌉,🎠,🎡,⛲️,🎢,🚢,⛵️,🚤,🚣,⚓️,🚀,✈️,💺,🚁,🚂,🚊,🚉,🚞,🚆,🚄,🚅,🚈,🚇,🚝,🚋,🚃,🚎,🚌,🚍,🚙,🚘,🚗,🚕,🚖,🚛,🚚,🚨,🚓,🚔,🚒,🚑,🚐,🚲,🚡,🚟,🚠,🚜,💈,🚏,🎫,🚦,🚥,⚠️,🚧,🔰,⛽️,🏮,🎰,♨️,🗿,🎪,🎭,📍,🚩,🇯🇵,🇰🇷,🇩🇪,🇨🇳,🇺🇸,🇫🇷,🇪🇸,🇮🇹,🇷🇺,🇬🇧"
+      }
+    },
+    {
+      "CVDataTitle" : "EmojiCategory-Symbols",
+      "CVCategoryImage" : "Emoji-SymbolImage",
+      "CVCategoryData" : {
+        "CVSkipNullGlyphs" : true,
+        "Data" : "1️⃣,2️⃣,3️⃣,4️⃣,5️⃣,6️⃣,7️⃣,8️⃣,9️⃣,0️⃣,🔟,🔢,#️⃣,🔣,⬆️,⬇️,⬅️,➡️,🔠,🔡,🔤,↗️,↖️,↘️,↙️,↔️,↕️,🔄,◀️,▶️,🔼,🔽,↩️,↪️,ℹ️,⏪,⏩,⏫,⏬,⤵️,⤴️,🆗,🔀,🔁,🔂,🆕,🆙,🆒,🆓,🆖,📶,🎦,🈁,🈯️,🈳,🈵,🈴,🈲,🉐,🈹,🈺,🈶,🈚️,🚻,🚹,🚺,🚼,🚾,🚰,🚮,🅿️,♿️,🚭,🈷,🈸,🈂,Ⓜ️,🛂,🛄,🛅,🛃,🉑,㊙️,㊗️,🆑,🆘,🆔,🚫,🔞,📵,🚯,🚱,🚳,🚷,🚸,⛔️,✳️,❇️,❎,✅,✴️,💟,🆚,📳,📴,🅰,🅱,🆎,🅾,💠,➿,♻️,♈️,♉️,♊️,♋️,♌️,♍️,♎️,♏️,♐️,♑️,♒️,♓️,⛎,🔯,🏧,💹,💲,💱,©,®,™,❌,‼️,⁉️,❗️,❓,❕,❔,⭕️,🔝,🔚,🔙,🔛,🔜,🔃,🕛,🕧,🕐,🕜,🕑,🕝,🕒,🕞,🕓,🕟,🕔,🕠,🕕,🕖,🕗,🕘,🕙,🕚,🕡,🕢,🕣,🕤,🕥,🕦,✖️,➕,➖,➗,♠️,♥️,♣️,♦️,💮,💯,✔️,☑️,🔘,🔗,➰,〰,〽️,🔱,◼️,◻️,◾️,◽️,▪️,▫️,🔺,🔲,🔳,⚫️,⚪️,🔴,🔵,🔻,⬜️,⬛️,🔶,🔷,🔸,🔹"
+      }
+    }
+  ],
+  "CVViewFontList" : [
+    "AppleColorEmoji"
+  ]
+}

db/aliases.applescript

@@ -0,0 +1,13 @@
+set jsCode to "document.getElementById('output').value"
+set json to missing value
+
+tell application "Safari"
+  repeat
+    set json to (do JavaScript jsCode in current tab of window 1)
+    if (json is not missing value) then exit repeat
+    delay 0.5
+  end repeat
+  close current tab of window 1
+end tell
+
+return json

db/aliases.html

@@ -0,0 +1,81 @@
+<!DOCTYPE html>
+<title>Emoji alias detection</title>
+<style>
+  textarea {
+    font-family: monospace;
+  }
+</style>
+
+<p>Save the following as <tt>emoji.json</tt>:</p>
+
+<textarea id="output" rows="50" cols="80"></textarea>
+
+<script>
+  const VARIATION_SELECTOR_15 = String.fromCharCode(0xfe0e);
+  const VARIATION_SELECTOR_16 = String.fromCharCode(0xfe0f);
+  const EMOJI_SIZE = 32
+
+  function detectAliases(db) {
+    for (var i = 0; i < db.length; ++i) {
+      var emoji = db[i];
+      var raw = emoji.emoji;
+      if (!raw) {
+        continue;
+      }
+
+      if (raw.indexOf(VARIATION_SELECTOR_16) > -1) {
+        var candidates = [raw.replace(VARIATION_SELECTOR_16, ""), raw];
+      } else {
+        var candidates = [raw, raw + VARIATION_SELECTOR_16];
+      }
+
+      var aliases = candidates.filter(isColorEmoji);
+      emoji.emoji = aliases[0];
+    }
+
+    dump(db);
+  }
+
+  function isColorEmoji(candidate) {
+    // Draw the emoji twice using a different color each time. If the emoji
+    // draws as the same color regardless of what color we set, it's a color
+    // emoji.
+    return color(candidate, "#f00") === color(candidate, "#0f0");
+  }
+
+  var canvas = document.createElement("canvas");
+  canvas.width = canvas.height = EMOJI_SIZE;
+
+  function color(emoji, rgb) {
+    var context = canvas.getContext("2d");
+    context.clearRect(0, 0, canvas.width, canvas.height);
+    context.fillStyle = rgb;
+    context.textBaseline = "top";
+    context.font = EMOJI_SIZE+"px Arial";
+    context.fillText(emoji, 0, 0);
+    var data = context.getImageData(0, 0, EMOJI_SIZE, EMOJI_SIZE).data;
+    for (var i = 0; i < data.length; i += 4) {
+      if (data[i] === 0 && data[i + 1] === 0 && data[i + 2] === 0) {
+        continue;
+      }
+      return data[i].toString(16)
+        + data[i + 1].toString(16)
+        + data[i + 2].toString(16);
+    }
+    return "no colored pixel found";
+  }
+
+  function dump(db) {
+    var json = JSON.stringify(db, null, "  ")
+      .replace(/^( +)(.+)\[\](,?)$/mg, "$1$2[\n$1]$3")
+      .replace(/,\n( *)  /g, "\n$1, ");
+    document.getElementById("output").value = json + "\n";
+  }
+
+  var xhr = new XMLHttpRequest;
+  xhr.onload = function() {
+    detectAliases(JSON.parse(this.responseText));
+  };
+  xhr.open("GET", "emoji.json", false);
+  xhr.send(null);
+</script>

db/dump.rb

@@ -1,69 +1,74 @@
-# frozen_string_literal: true
-
-require "i18n"
 require 'emoji'
 require 'json'
-require_relative './emoji-test-parser'
 
-I18n.config.available_locales = :en
-items = []
+names_list = File.expand_path('../NamesList.txt', __FILE__)
 
-_, categories = EmojiTestParser.parse(File.expand_path("../../vendor/unicode-emoji-test.txt", __FILE__))
-seen_existing = {}
+class UnicodeCharacter
+  attr_reader :code, :description, :aliases
 
-for category in categories
-  for sub_category in category[:emoji]
-    for emoji_item in sub_category[:emoji]
-      raw = emoji_item[:sequences][0]
-      existing_emoji = Emoji.find_by_unicode(raw) || Emoji.find_by_unicode("#{raw}\u{fe0f}")
-      if seen_existing.key?(existing_emoji)
-        existing_emoji = nil
-      else
-        seen_existing[existing_emoji] = true
-      end
-      description = emoji_item[:description].sub(/^E\d+(\.\d+)? /, '')
-      output_item = {
-        emoji: raw,
-        description: description,
-        category: category[:name],
-      }
-      if existing_emoji
-        output_item.update(
-          aliases: existing_emoji.aliases,
-          tags: existing_emoji.tags,
-          unicode_version: existing_emoji.unicode_version,
-          ios_version: existing_emoji.ios_version,
-        )
-      else
-        output_item.update(
-          aliases: [I18n.transliterate(description).gsub(/\W+/, '_').downcase],
-          tags: [],
-          unicode_version: "15.0",
-          ios_version: "16.4",
-        )
-      end
-      output_item[:skin_tones] = true if emoji_item[:skin_tones]
-      items << output_item
+  @index = {}
+  class << self
+    attr_reader :index
+
+    def fetch(code, *args, &block)
+      code = code.to_s(16).rjust(4, '0') if code.is_a?(Integer)
+      index.fetch(code, *args, &block)
     end
   end
+
+  def initialize(code, description)
+    @code = code.downcase
+    @description = description.downcase
+    @aliases = []
+    @references = []
+
+    self.class.index[@code] = self
+  end
+
+  def add_alias(string)
+    @aliases.concat string.split(/\s*,\s*/)
+  end
+
+  def add_reference(code)
+    @references << code.downcase
+  end
 end
 
-missing_emoji = Emoji.all.reject { |e| e.custom? || seen_existing.key?(e) }
-if missing_emoji.any?
-  $stderr.puts "Error: these `emoji.json` entries were not matched:"
-  $stderr.puts missing_emoji.map { |e| "%s (%s)" % [e.hex_inspect, e.name] }
-  exit 1
-end
+char = nil
 
-for emoji in Emoji.all.select(&:custom?)
-  items << {
-    aliases: emoji.aliases,
-    tags: emoji.tags,
-  }
+File.foreach(names_list) do |line|
+  case line
+  when /^[A-F0-9]{4,5}\t/
+    code, desc = line.chomp.split("\t", 2)
+    codepoint = code.hex
+    char = UnicodeCharacter.new(code, desc)
+  when /^\t= /
+    char.add_alias($')
+  when /^\tx .+ - ([A-F0-9]{4,5})\)$/
+    char.add_reference($1)
+  end
 end
 
 trap(:PIPE) { abort }
 
+items = []
+variation_codepoint = Emoji::VARIATION_SELECTOR_16.codepoints[0]
+
+for emoji in Emoji.all
+  item = {}
+
+  unless emoji.custom?
+    chars = emoji.raw.codepoints.map { |code| UnicodeCharacter.fetch(code) unless code == variation_codepoint }.compact
+    item[:emoji] = emoji.raw
+    item[:description] = chars.map(&:description).join(' + ')
+  end
+
+  item[:aliases] = emoji.aliases
+  item[:tags] = emoji.tags
+
+  items << item
+end
+
 puts JSON.pretty_generate(items)
   .gsub("\n\n", "\n")
   .gsub(/,\n( +)/) { "\n%s, " % $1[2..-1] }

db/emoji-test-parser.rb

@@ -1,117 +0,0 @@
-# frozen_string_literal: true
-
-module EmojiTestParser
-  VARIATION_SELECTOR_16 = "\u{fe0f}"
-  SKIN_TONES = [
-    "\u{1F3FB}", # light skin tone
-    "\u{1F3FC}", # medium-light skin tone
-    "\u{1F3FD}", # medium skin tone
-    "\u{1F3FE}", # medium-dark skin tone
-    "\u{1F3FF}", # dark skin tone
-  ]
-  SKIN_TONES_RE = /(#{SKIN_TONES.join("|")})/o
-  SKIP_TYPES = ["unqualified", "component"]
-
-  module_function
-
-  def parse(filename)
-    File.open(filename, "r:UTF-8") do |file|
-      parse_file(file)
-    end
-  end
-
-  def parse_file(io)
-    data = []
-    emoji_map = {}
-    category = nil
-    sub_category = nil
-
-    io.each do |line|
-      begin
-        if line.start_with?("# group: ")
-          _, group_name = line.split(":", 2)
-          category = {
-            name: group_name.strip,
-            emoji: [],
-          }
-          data << category
-          sub_category = nil
-        elsif line.start_with?("# subgroup: ")
-          _, group_name = line.split(":", 2)
-          sub_category = {
-            name: group_name.strip,
-            emoji: [],
-          }
-          category[:emoji] << sub_category
-        elsif line.start_with?("#") || line.strip.empty?
-          next
-        else
-          row, desc = line.split("#", 2)
-          desc = desc.strip.split(" ", 2)[1]
-          codepoints, qualification = row.split(";", 2)
-          next if SKIP_TYPES.include?(qualification.strip)
-          emoji_raw = codepoints.strip.split.map { |c| c.hex }.pack("U*")
-          emoji_normalized = emoji_raw
-            .gsub(VARIATION_SELECTOR_16, "")
-            .gsub(SKIN_TONES_RE, "")
-          emoji_item = emoji_map[emoji_normalized]
-          if SKIN_TONES.any? { |s| emoji_raw.include?(s) }
-            emoji_item[:skin_tones] = true if emoji_item
-            next
-          end
-          if emoji_item
-            emoji_item[:sequences] << emoji_raw
-          else
-            emoji_item = {
-              sequences: [emoji_raw],
-              description: desc,
-            }
-            emoji_map[emoji_normalized] = emoji_item
-            sub_category[:emoji] << emoji_item
-          end
-        end
-      rescue
-        warn "line: %p" % line
-        raise
-      end
-    end
-
-    [emoji_map, data]
-  end
-end
-
-if $0 == __FILE__
-  html_output = false
-  if ARGV[0] == "--html"
-    ARGV.shift
-    html_output = true
-  end
-
-  _, categories = EmojiTestParser.parse(File.expand_path("../../vendor/unicode-emoji-test.txt", __FILE__))
-
-  trap(:PIPE) { abort }
-
-  if html_output
-    puts "<!doctype html>"
-    puts "<meta charset=utf-8>"
-    for category in categories
-      puts "<h2>#{category[:name]}</h2>"
-      for sub_category in category[:emoji]
-        puts "<h3>#{sub_category[:name]}</h3>"
-        puts "<ol>"
-        for char in sub_category[:emoji]
-          puts "<li>"
-          for sequence in char[:sequences]
-            codepoints = sequence.unpack("U*").map { |c| c.to_s(16).upcase }.join(" ")
-            printf '<span class=emoji title="%s">%s</span> ', codepoints, sequence
-          end
-          puts "#{char[:description]}</li>"
-        end
-        puts "</ol>"
-      end
-    end
-  else
-    require "json"
-    puts JSON.pretty_generate(categories)
-  end
-end

db/index.html

@@ -66,7 +66,7 @@ li > span {
       if (emoji.emoji) els[0].textContent = emoji.emoji
       else {
         var img = document.createElement('img')
-        img.src = "../images/" + emoji.aliases[0] + ".png"
+        img.src = "../images/emoji/" + emoji.aliases[0] + ".png"
         els[0].appendChild(img)
       }
       els[1].textContent = emoji.description || ''

ext/gemoji/emoji.h

@@ -0,0 +1,11 @@
+static const long emoji_byte_lengths[][5] = {
+	{0},
+	{11, 8, 7, 4, 0},
+	{6, 3, 0},
+	{7, 0},
+	{6, 0},
+	{5, 0},
+};
+
+static const int8_t emoji_magic_bytes[] =
+{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};

ext/gemoji/extconf.rb

@@ -0,0 +1,6 @@
+require 'mkmf'
+
+$CFLAGS << ' -ggdb3 -O0 '
+
+dir_config('gemoji')
+create_makefile('gemoji')

ext/gemoji/gemoji.c

@@ -0,0 +1,125 @@
+#include <ruby.h>
+#include <ruby/encoding.h>
+#include "emoji.h"
+
+#define unlikely(x) __builtin_expect((x),0)
+
+static long
+lookup_emoji(VALUE *rb_emoji, VALUE rb_bytes, VALUE rb_unicode_map, const long *possible_len)
+{
+	const uint8_t *src = (uint8_t *)RSTRING_PTR(rb_bytes);
+
+	for (; *possible_len; ++possible_len) {
+		const long emoji_size = *possible_len;
+
+		if (emoji_size > RSTRING_LEN(rb_bytes))
+			continue;
+
+		if ((src[emoji_size - 1] & 0xC0) != 0x80)
+			continue;
+
+		rb_str_set_len(rb_bytes, emoji_size);
+
+		*rb_emoji = rb_hash_lookup(rb_unicode_map, rb_bytes);
+		if (!NIL_P(*rb_emoji))
+			return emoji_size;
+	}
+
+	return 0;
+}
+
+static VALUE
+replace_emoji(const uint8_t *src, long size, VALUE rb_unicode_map)
+{
+	VALUE rb_emoji, rb_bytes, rb_out = Qnil;
+	long i = 0, org, emoji_len;
+	int8_t emoji_byte;
+
+	while (i < size) {
+		org = i;
+
+retry_search:
+		while (i < size && (emoji_byte = emoji_magic_bytes[(int)src[i]]) == 0)
+			i++;
+
+		if (i + 1 < size && (src[i + 1] & 0x80) != 0x80) {
+			i++;
+			goto retry_search;
+		}
+
+		if (unlikely(org == 0)) {
+			if (i == size)
+				return Qnil;
+
+			rb_out = rb_str_buf_new(size * 4 / 3);
+			rb_enc_associate(rb_out, rb_utf8_encoding());
+
+			rb_bytes = rb_str_buf_new(16);
+			rb_enc_associate(rb_bytes, rb_utf8_encoding());
+		}
+
+		if (i > org)
+			rb_str_buf_cat(rb_out, (const char *)src + org, i - org);
+
+		if (unlikely(i == size))
+			break;
+
+		emoji_len = size - i;
+		if (emoji_len > 12)
+			emoji_len = 12;
+
+		memcpy(RSTRING_PTR(rb_bytes), src + i, emoji_len);
+		rb_str_set_len(rb_bytes, emoji_len);
+
+		emoji_len = lookup_emoji(&rb_emoji, rb_bytes, rb_unicode_map, emoji_byte_lengths[(int)emoji_byte]);
+
+		if (emoji_len) {
+			VALUE rb_repl = rb_yield(rb_emoji);
+
+			if (NIL_P(rb_repl)) {
+				rb_str_buf_cat(rb_out, (const char *)src + i, emoji_len);
+			} else {
+				Check_Type(rb_repl, T_STRING);
+				rb_str_buf_append(rb_out, rb_repl);
+			}
+
+			i += emoji_len;
+			continue;
+		}
+
+		rb_str_buf_cat(rb_out, (const char *)src + i, 1);
+		i++;
+	}
+
+	return rb_out;
+}
+
+static VALUE
+rb_gemoji_replace_unicode(VALUE klass, VALUE rb_source)
+{
+	VALUE rb_output;
+	VALUE rb_unicode_map = rb_funcall(klass, rb_intern("unicodes_index"), 0);
+
+	Check_Type(rb_source, T_STRING);
+	Check_Type(rb_unicode_map, T_HASH);
+
+	rb_must_asciicompat(rb_source);
+
+	if (ENC_CODERANGE_ASCIIONLY(rb_source))
+		return rb_source;
+
+	if (rb_enc_get(rb_source) != rb_utf8_encoding())
+		rb_raise(rb_eEncCompatError, "expected UTF-8 encoding");
+
+	rb_output = replace_emoji((uint8_t *)RSTRING_PTR(rb_source), RSTRING_LEN(rb_source), rb_unicode_map);
+	if (NIL_P(rb_output))
+		return rb_source;
+
+	return rb_output;
+}
+
+void Init_gemoji(void)
+{
+	VALUE rb_mEmoji = rb_define_module("Emoji");
+	rb_define_method(rb_mEmoji, "gsub_unicode", rb_gemoji_replace_unicode, 1);
+}

gemoji.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name    = "gemoji"
-  s.version = "4.1.0"
-  s.summary = "Unicode emoji library"
-  s.description = "Character information and metadata for Unicode emoji."
+  s.version = "2.1.0"
+  s.summary = "Emoji conversion and image assets"
+  s.description = "Image assets and character information for emoji."
 
   s.required_ruby_version = '> 1.9'
 
@@ -13,8 +13,9 @@ Gem::Specification.new do |s|
 
   s.files = Dir[
     "README.md",
-    "LICENSE",
+    "images/**/*.png",
     "db/emoji.json",
     "lib/**/*.rb",
+    "lib/tasks/*.rake"
   ]
 end