forked from TensorLayer/tensorlayer3
81 lines
2.3 KiB
Python
81 lines
2.3 KiB
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
import unittest
|
|
|
|
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
|
|
|
|
import tensorflow as tf
|
|
import tensorlayer as tl
|
|
|
|
from tensorflow.python.platform import gfile
|
|
from tests.utils import CustomTestCase
|
|
import nltk
|
|
nltk.download('punkt')
|
|
|
|
|
|
class Test_Leaky_ReLUs(CustomTestCase):
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
pass
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
pass
|
|
|
|
def test_as_bytes(self):
|
|
origin_str = "str"
|
|
origin_bytes = b'bytes'
|
|
converted_str = tl.nlp.as_bytes(origin_str)
|
|
converted_bytes = tl.nlp.as_bytes(origin_bytes)
|
|
print('str after using as_bytes:', converted_str)
|
|
print('bytes after using as_bytes:', converted_bytes)
|
|
|
|
def test_as_text(self):
|
|
origin_str = "str"
|
|
origin_bytes = b'bytes'
|
|
converted_str = tl.nlp.as_text(origin_str)
|
|
converted_bytes = tl.nlp.as_text(origin_bytes)
|
|
print('str after using as_text:', converted_str)
|
|
print('bytes after using as_text:', converted_bytes)
|
|
|
|
def test_save_vocab(self):
|
|
words = tl.files.load_matt_mahoney_text8_dataset()
|
|
vocabulary_size = 50000
|
|
data, count, dictionary, reverse_dictionary = tl.nlp.build_words_dataset(words, vocabulary_size, True)
|
|
tl.nlp.save_vocab(count, name='vocab_text8.txt')
|
|
|
|
def test_basic_tokenizer(self):
|
|
c = "how are you?"
|
|
tokens = tl.nlp.basic_tokenizer(c)
|
|
print(tokens)
|
|
|
|
def test_generate_skip_gram_batch(self):
|
|
data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
|
|
batch, labels, data_index = tl.nlp.generate_skip_gram_batch(
|
|
data=data, batch_size=8, num_skips=2, skip_window=1, data_index=0
|
|
)
|
|
print(batch)
|
|
print(labels)
|
|
|
|
def test_process_sentence(self):
|
|
c = "how are you?"
|
|
c = tl.nlp.process_sentence(c)
|
|
print(c)
|
|
|
|
def test_words_to_word_id(self):
|
|
words = tl.files.load_matt_mahoney_text8_dataset()
|
|
vocabulary_size = 50000
|
|
data, count, dictionary, reverse_dictionary = tl.nlp.build_words_dataset(words, vocabulary_size, True)
|
|
ids = tl.nlp.words_to_word_ids(words, dictionary)
|
|
context = tl.nlp.word_ids_to_words(ids, reverse_dictionary)
|
|
# print(ids)
|
|
# print(context)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
unittest.main()
|