[
  {
    "Bit per Character (BPC)": 0.98,
    "Number of params": "1542M",
    "code_links": [
      {
        "title": "huggingface/transformers",
        "url": "https://github.com/huggingface/transformers"
      },
      {
        "title": "openai/gpt-2",
        "url": "https://github.com/openai/gpt-2"
      },
      {
        "title": "minimaxir/gpt-2-simple",
        "url": "https://github.com/minimaxir/gpt-2-simple"
      },
      {
        "title": "huggingface/swift-coreml-transformers",
        "url": "https://github.com/huggingface/swift-coreml-transformers"
      },
      {
        "title": "imcaspar/gpt2-ml",
        "url": "https://github.com/imcaspar/gpt2-ml"
      },
      {
        "title": "akanyaani/gpt-2-tensorflow2.0",
        "url": "https://github.com/akanyaani/gpt-2-tensorflow2.0"
      }
    ],
    "date": "2019-02-14",
    "date2": 20190214,
    "model": "GPT-2",
    "paper": {
      "title": "Language Models are Unsupervised Multitask Learners",
      "url": "https://cknow.io/lib/2a5eff987ab8d8ec"
    },
    "paper_data_uoa": "2a5eff987ab8d8ec"
  },
  {
    "Bit per Character (BPC)": 1.07,
    "Number of params": "209M",
    "code_links": [
      {
        "title": "facebookresearch/adaptive-span",
        "url": "https://github.com/facebookresearch/adaptive-span"
      },
      {
        "title": "ofirpress/sandwich_transformer",
        "url": "https://github.com/ofirpress/sandwich_transformer"
      },
      {
        "title": "jerrodparker20/adaptive-transformers-in-rl",
        "url": "https://github.com/jerrodparker20/adaptive-transformers-in-rl"
      },
      {
        "title": "prajjwal1/adaptive_transformer",
        "url": "https://github.com/prajjwal1/adaptive_transformer"
      },
      {
        "title": "JoeRoussy/adaptive-attention-in-cv",
        "url": "https://github.com/JoeRoussy/adaptive-attention-in-cv"
      },
      {
        "title": "prajjwal1/fluence",
        "url": "https://github.com/prajjwal1/fluence"
      }
    ],
    "date": "2019-05-19",
    "date2": 20190519,
    "model": "24L Transformer + 8K adaptive span",
    "paper": {
      "title": "Adaptive Attention Span in Transformers",
      "url": "https://cknow.io/lib/94ad0986ed638bb1"
    },
    "paper_data_uoa": "94ad0986ed638bb1"
  },
  {
    "Bit per Character (BPC)": 1.08,
    "Number of params": "277M",
    "code_links": [
      {
        "title": "huggingface/transformers",
        "url": "https://github.com/huggingface/transformers"
      },
      {
        "title": "kimiyoung/transformer-xl",
        "url": "https://github.com/kimiyoung/transformer-xl"
      },
      {
        "title": "benkrause/dynamiceval-transformer",
        "url": "https://github.com/benkrause/dynamiceval-transformer"
      },
      {
        "title": "Machine-Learning-Tokyo/Poetry-GAN",
        "url": "https://github.com/Machine-Learning-Tokyo/Poetry-GAN"
      },
      {
        "title": "inzva/fake-academic-paper-generation",
        "url": "https://github.com/inzva/fake-academic-paper-generation"
      },
      {
        "title": "wxt1997/Transformer-Transducer",
        "url": "https://github.com/wxt1997/Transformer-Transducer"
      },
      {
        "title": "threelittlemonkeys/transformer-pytorch",
        "url": "https://github.com/threelittlemonkeys/transformer-pytorch"
      },
      {
        "title": "cmunnis/BERT_vs_Transformer-XL",
        "url": "https://github.com/cmunnis/BERT_vs_Transformer-XL"
      },
      {
        "title": "zhdbwe/Paper-DailyReading",
        "url": "https://github.com/zhdbwe/Paper-DailyReading"
      },
      {
        "title": "listenviolet/XLNet",
        "url": "https://github.com/listenviolet/XLNet"
      },
      {
        "title": "samwisegamjeee/pytorch-transformers",
        "url": "https://github.com/samwisegamjeee/pytorch-transformers"
      }
    ],
    "date": "2019-01-09",
    "date2": 20190109,
    "model": "Transformer-XL - 24 layers",
    "paper": {
      "title": "Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context",
      "url": "https://cknow.io/lib/05ebe08ab5d3e9ea"
    },
    "paper_data_uoa": "05ebe08ab5d3e9ea"
  },
  {
    "Bit per Character (BPC)": 1.08,
    "Number of params": "114M",
    "code_links": [
      {
        "title": "lucidrains/reformer-pytorch",
        "url": "https://github.com/lucidrains/reformer-pytorch"
      },
      {
        "title": "facebookresearch/adaptive-span",
        "url": "https://github.com/facebookresearch/adaptive-span"
      },
      {
        "title": "lucidrains/routing-transformer",
        "url": "https://github.com/lucidrains/routing-transformer"
      },
      {
        "title": "lucidrains/memory-transformer-xl",
        "url": "https://github.com/lucidrains/memory-transformer-xl"
      }
    ],
    "date": "2019-07-02",
    "date2": 20190702,
    "model": "All-attention network - 36 layers",
    "paper": {
      "title": "Augmenting Self-attention with Persistent Memory",
      "url": "https://cknow.io/lib/bbd24632af316e45"
    },
    "paper_data_uoa": "bbd24632af316e45"
  },
  {
    "Bit per Character (BPC)": 1.11,
    "Number of params": "38M",
    "code_links": [
      {
        "title": "facebookresearch/adaptive-span",
        "url": "https://github.com/facebookresearch/adaptive-span"
      },
      {
        "title": "ofirpress/sandwich_transformer",
        "url": "https://github.com/ofirpress/sandwich_transformer"
      },
      {
        "title": "jerrodparker20/adaptive-transformers-in-rl",
        "url": "https://github.com/jerrodparker20/adaptive-transformers-in-rl"
      },
      {
        "title": "prajjwal1/adaptive_transformer",
        "url": "https://github.com/prajjwal1/adaptive_transformer"
      },
      {
        "title": "JoeRoussy/adaptive-attention-in-cv",
        "url": "https://github.com/JoeRoussy/adaptive-attention-in-cv"
      },
      {
        "title": "prajjwal1/fluence",
        "url": "https://github.com/prajjwal1/fluence"
      }
    ],
    "date": "2019-05-19",
    "date2": 20190519,
    "model": "12L Transformer + 8K adaptive span",
    "paper": {
      "title": "Adaptive Attention Span in Transformers",
      "url": "https://cknow.io/lib/94ad0986ed638bb1"
    },
    "paper_data_uoa": "94ad0986ed638bb1"
  },
  {
    "Bit per Character (BPC)": 1.11,
    "Number of params": "38M",
    "code_links": [
      {
        "title": "lucidrains/reformer-pytorch",
        "url": "https://github.com/lucidrains/reformer-pytorch"
      },
      {
        "title": "facebookresearch/adaptive-span",
        "url": "https://github.com/facebookresearch/adaptive-span"
      },
      {
        "title": "lucidrains/routing-transformer",
        "url": "https://github.com/lucidrains/routing-transformer"
      },
      {
        "title": "lucidrains/memory-transformer-xl",
        "url": "https://github.com/lucidrains/memory-transformer-xl"
      }
    ],
    "date": "2019-07-02",
    "date2": 20190702,
    "model": "All-attention network - 18 layers",
    "paper": {
      "title": "Augmenting Self-attention with Persistent Memory",
      "url": "https://cknow.io/lib/bbd24632af316e45"
    },
    "paper_data_uoa": "bbd24632af316e45"
  },
  {
    "Bit per Character (BPC)": 1.27,
    "Number of params": "46M",
    "code_links": [
      {
        "title": "julian121266/RecurrentHighwayNetworks",
        "url": "https://github.com/julian121266/RecurrentHighwayNetworks"
      },
      {
        "title": "jzilly/RecurrentHighwayNetworks",
        "url": "https://github.com/jzilly/RecurrentHighwayNetworks"
      },
      {
        "title": "vermaMachineLearning/Pytorch-JIT-Recurrent-Highway-Network",
        "url": "https://github.com/vermaMachineLearning/Pytorch-JIT-Recurrent-Highway-Network"
      },
      {
        "title": "davidsvaughn/dts-tf",
        "url": "https://github.com/davidsvaughn/dts-tf"
      }
    ],
    "date": "2016-07-12",
    "date2": 20160712,
    "model": "Large RHN",
    "paper": {
      "title": "Recurrent Highway Networks",
      "url": "https://cknow.io/lib/f8a413cba9ae8258"
    },
    "paper_data_uoa": "f8a413cba9ae8258"
  }
]