CLTK Word Tokenization (Latin NLP with Python 11)

Published: 04 September 2020
on channel: Python Tutorials for Digital Humanities

569

The Code:
from cltk.stem.latin.j_v import JVReplacer
from cltk.stem.latin.declension import CollatinusDecliner
from cltk.lemmatize.latin.backoff import BackoffLatinLemmatizer
from cltk.prosody.latin.macronizer import Macronizer
from cltk.tokenize.line import LineTokenizer
from cltk.tokenize.latin.sentence import SentenceTokenizer
from cltk.corpus.utils.formatter import remove_non_latin
from cltk.tokenize.word import WordTokenizer
import re

with open ('data/pl.txt', "r") as f:
text = f.read()

def jvtext(text):
j = JVReplacer()
text = j.replace(text)
return (text)

def clean_pl(text, lower=False):
cleaned = re.sub(r"[\(\[].*?[\)\]]", "", text)
cleaned = cleaned.replace(" ", " ").replace(" ", " ")
if lower==True:
lower_cleaned = cleaned.lower()
return (cleaned, lower_cleaned)
return(cleaned)

def dec(words):
decliner = CollatinusDecliner()
dec_words = {}
try:
for word in words:
dec_word = decliner.decline(word)
dec_words[word] = dec_word
except:
Exception
return (dec_words)

def lemma(tokens):
lemmatizer = BackoffLatinLemmatizer()
tokens = lemmatizer.lemmatize(tokens)
return (tokens)

def macron(text):
macronizer = Macronizer("tag_ngram_123_backoff")
text = macronizer.macronize_text(text)
return (text)

def line_tok(text):
tokenizer = LineTokenizer("latin")
text = tokenizer.tokenize(text)
return (text)

def sent_tok(text, punct=True):
sent_tokenizer = SentenceTokenizer()
sentences = sent_tokenizer.tokenize(text)
new_sents = []
if punct==True:
for sent in sentences:
sent = remove_non_latin(sent)
sent = sent.lower()
new_sents.append(sent)
return(new_sents)
return (sentences)

def word_tok(text):
word_tokenizer = WordTokenizer("latin")
words = word_tokenizer.tokenize(text)
return (words)

text = clean_pl(text)
text = jvtext(text)
sentences = sent_tok(text, punct=True)
sentence = sentences[0]
words = word_tok(sentence)
words = lemma(words)

new = []
for word in words:
new.append(word[1])
decs = dec(new)

If you enjoy this video, please subscribe. I provide all my content at no cost. If you want to support my channel, please donate via
PayPal: https://www.paypal.com/cgi-bin/webscr...
Patreon: / wjbmattingly (its my www.themedievalworld.com account as well).

If there's a specific video you would like to see or a tutorial series, let me know in the comments and I will try and make it.

If you liked this video, check out www.PythonHumanities.com, where I have Coding Exercises, Lessons, on-site Python shells where you can experiment with code, and a text version of the material discussed here.

You can follow me at:
/ wjb_mattingly

On this page of the site you can watch the video online CLTK Word Tokenization (Latin NLP with Python 11) with a duration of hours minute second in good quality, which was uploaded by the user Python Tutorials for Digital Humanities 04 September 2020, share the link with friends and acquaintances, this video has already been watched 569 times on youtube and it was liked by 8 viewers. Enjoy your viewing!

1,146