const str_objs = [{str: 'abc'}, {str: 'def'}, {str: 'ghi'}];
const strs = str_objs.map((x) => { return x.str; });
console.log(strs);
[ 'abc', 'def', 'ghi' ]
sudo yum install libjpeg-turbo-devel sudo yum install zlib-devel sudo yum install python36-devel sudo pip install pillow
sudo yum install -y epel-release sudo yum groupinstall -y Xfce
import requests
from PIL import Image
from io import BytesIO
img_url = 'https://i.xgoo.jp/img/static/global/cmm/sn/logo_gooblog.png'
uto=format&w=384'
res = requests.get(img_url)
inst = BytesIO(res.content)
img_obj = Image.open(inst).convert('RGB')
inst.close()
img_obj.save('img.png')
# -*- coding:utf-8 -*-
import sys
import json
import torch
import torchvision.transforms as transforms
from PIL import Image
from googlenet_pytorch import GoogLeNet
feature_vector = None
def get_feature_vector(preproc, model, img_file):
input_image = Image.open(img_file)
input_tensor = preproc(input_image)
input_batch = input_tensor.unsqueeze(0)
logits = model(input_batch)
preds = torch.topk(logits, k=5).indices.squeeze(0).tolist()
return feature_vector
def forward_hook(module, inputs, outputs):
global feature_vector
feature_vector = outputs.detach().clone()[0].tolist()
def init():
preproc = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
labels_map = json.load(open("labels_map.txt"))
labels_map = [labels_map[str(i)] for i in range(1000)]
model = GoogLeNet.from_pretrained("googlenet")
model.eval()
layers = list(model.children())
handle = layers[-2].register_forward_hook(forward_hook)
return preproc, model
def main():
preproc, model = init()
img_file = sys.argv[1]
fv = get_feature_vector(preproc, model, img_file)
print(fv)
return 0
if __name__ == '__main__':
res = main()
exit(res)
[appstream] name=CentOS Linux $releasever - AppStream baseurl=http://linuxsoft.cern.ch/centos-vault/8.4.2105/AppStream/$basearch/os/ #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=AppStream&infra=$infra #baseurl=http://mirror.centos.org/$contentdir/$releasever/AppStream/$basearch/os/ gpgcheck=1 enabled=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial
import base64
data = 'あいうえお'
data_enc = data.encode('utf-8')
data_b64 = base64.b64encode(data_enc)
print(data_b64)
data_b64_dec = base64.b64decode(data_b64)
data_dec = data_b64_dec.decode('utf-8')
print(data_dec)
b'44GC44GE44GG44GI44GK' あいうえお
■algorithm=lbfgs, c1=0.1, c2=0.1
crf = sklearn_crfsuite.CRF(
algorithm='lbfgs',
c1=0.1,
c2=0.1,
max_iterations=100,
all_possible_transitions=True
)
0.9441077916505949
■algorithm=lbfgs, c1=0.5, c2=0.5
crf = sklearn_crfsuite.CRF(
algorithm='lbfgs',
c1=0.5,
c2=0.5,
max_iterations=100,
all_possible_transitions=True
)
0.945220761760303
■algorithm=l2sgd, c2=0.1
crf = sklearn_crfsuite.CRF(
algorithm='l2sgd',
c2=0.1,
max_iterations=100,
all_possible_transitions=True
)
0.9426298984809508
■algorithm=l2sgd, c2=0.5
crf = sklearn_crfsuite.CRF(
algorithm='l2sgd',
c2=0.5,
max_iterations=100,
all_possible_transitions=True
)
0.9449389892604881
■algorithm=ap
crf = sklearn_crfsuite.CRF(
algorithm='ap',
max_iterations=100,
all_possible_transitions=True
)
0.9418937778719487
■algorithm=pa
crf = sklearn_crfsuite.CRF(
algorithm='pa',
max_iterations=100,
all_possible_transitions=True
)
0.9407907098905913
■algorithm=arow
crf = sklearn_crfsuite.CRF(
algorithm='arow',
max_iterations=100,
all_possible_transitions=True
)
0.8690576483344261
# 単語の特徴量
def word_featureh(sent, i):
word = sent[i]
tkn = word[0]
pos = word[1]
if i > 0:
word_p1 = sent[i-1]
tkn_p1 = word_p1[0]
pos_p1 = word_p1[1]
else:
tkn_p1 = ''
pos_p1 = ''
if i >= len(sent)-1:
tkn_n1 = '<e>'
pos_n1 = '<e>'
else:
word_n1 = sent[i+1]
tkn_n1 = word_n1[0]
pos_n1 = word_n1[1]
feath = {
'bias': 1.0,
'token': tkn.lower(),
'token.isuppser()': tkn.isupper(),
'token.istitle()': tkn.istitle(),
'token.isdigit()': tkn.isdigit(),
'pos': pos,
'token_p1': tkn_p1.lower(),
'token_p1.isuppser()': tkn_p1.isupper(),
'token_p1.istitle()': tkn_p1.istitle(),
'token_p1.isdigit()': tkn_p1.isdigit(),
'pos_p1': pos_p1,
'token_n1': tkn_n1.lower(),
'token_n1.isuppser()': tkn_n1.isupper(),
'token_n1.istitle()': tkn_n1.istitle(),
'token_n1.isdigit()': tkn_n1.isdigit(),
'pos_n1': pos_n1,
}
return feath
0.9618864951695193
[('Confidence', 'NN', 'B-NP'), ('in', 'IN', 'O'), ('the', 'DT', 'B-NP'), ('pound', 'NN', 'I-NP'), ('is', 'VBZ', 'O'), ('widely', 'RB', 'O'), ('expected', 'VBN', 'O'), ('to', 'TO', 'O'), ('take', 'VB', 'O'), ('another', 'DT', 'B-NP'), ('sharp', 'JJ', 'I-NP'), ('dive', 'NN', 'I-NP'), ('if', 'IN', 'O'), ('trade', 'NN', 'B-NP'), ('figures', 'NNS', 'I-NP'), ('for', 'IN', 'O'), ('September', 'NNP', 'B-NP'), (',', ',', 'O'), ('due', 'JJ', 'O'), ('for', 'IN', 'O'), ('release', 'NN', 'B-NP'), ('tomorrow', 'NN', 'B-NP'), (',', ',', 'O'), ('fail', 'VB', 'O'), ('to', 'TO', 'O'), ('show', 'VB', 'O'), ('a', 'DT', 'B-NP'), ('substantial', 'JJ', 'I-NP'), ('improvement', 'NN', 'I-NP'), ('from', 'IN', 'O'), ('July', 'NNP', 'B-NP'), ('and', 'CC', 'I-NP'), ('August', 'NNP', 'I-NP'), ("'s", 'POS', 'B-NP'), ('near-record', 'JJ', 'I-NP'), ('deficits', 'NNS', 'I-NP'), ('.', '.', 'O')]
import nltk
from nltk.corpus import conll2000
import sklearn
import sklearn_crfsuite
from sklearn_crfsuite import metrics
# [(token, pos, label), ...]
def create_data(file):
sents = conll2000.chunked_sents(file, chunk_types=['NP'])
sents = [nltk.chunk.tree2conlltags(s) for s in sents]
return sents
# 単語の特徴量
def word_featureh(sent, i):
word = sent[i]
tkn = word[0]
pos = word[1]
feath = {
'bias': 1.0,
'token': tkn.lower(),
'token.isuppser()': tkn.isupper(),
'token.istitle()': tkn.istitle(),
'token.isdigit()': tkn.isdigit(),
'pos': pos,
}
return feath
# 文の特徴量
def sent_features(sent):
return [word_featureh(sent, i) for i in range(len(sent))]
# 文のラベル
def sent_labels(sent):
return [word[2] for word in sent]
def main():
train_sents = create_data('train.txt')
X_train = [sent_features(s) for s in train_sents]
y_train = [sent_labels(s) for s in train_sents]
test_sents = create_data('test.txt')
X_test = [sent_features(s) for s in test_sents]
y_test = [sent_labels(s) for s in test_sents]
crf = sklearn_crfsuite.CRF(
algorithm='lbfgs',
c1=0.1,
c2=0.1,
max_iterations=100,
all_possible_transitions=True
)
crf.fit(X_train, y_train)
labels = list(crf.classes_)
labels.remove('O')
y_pred = crf.predict(X_test)
res = metrics.flat_f1_score(y_test, y_pred,
average='weighted', labels=labels)
print(res)
return 0
if __name__ == '__main__':
res = main()
exit(res)
0.9441077916505949
abc (abc|def) a(b|c)(d|e)fただし、(...) の中に (...) は書けないものとします。
CHAR: [^|()] ※|、(、) 以外の文字。 LB: ( RB: ) PIPE: |
import sys
import ply.lex as lex
tokens = (
'CHAR',
'LB',
'RB',
'PIPE',
)
t_CHAR = r'[^|()]'
t_LB = r'[(]'
t_RB = r'[)]'
t_PIPE = r'[|]'
def t_error(t):
sys.stderr.write('illegal char "%s"' % t.value[0])
t.lexer.sip(t)
lexer = lex.lex()
pttrn : sub_pttrns
sub_pttrns : sub_pttrn
| sub_pttrns sub_pttrn
sub_pttrn : chars
| select
select : LB chars RB
| LB chars select_iter RB
select_iter : PIPE chars
| PIPE chars select_iter
chars : CHAR
| chars CHAR
import sys
import ply.yacc as yacc
from test_lex import tokens
def p_pttrn(p):
'''pttrn : sub_pttrns'''
p[0] = p[1]
def p_sub_pttrns(p):
'''sub_pttrns : sub_pttrn
| sub_pttrns sub_pttrn'''
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[1]
p[0].extend(p[2])
def p_sub_pttrn(p):
'''sub_pttrn : chars
| select'''
p[0] = p[1]
def p_select_pttrn(p):
'''select : LB chars RB
| LB chars select_iter RB'''
if len(p) == 4:
p[0] = [{'or': [p[2]]}]
else:
p[0] = [p[2]]
p[0].extend(p[3])
p[0] = [{'or': p[0]}]
def p_select_iter(p):
'''select_iter : PIPE chars
| PIPE chars select_iter'''
if len(p) == 3:
p[0] = p[2]
else:
p[0] = [p[2]]
p[0].extend(p[3])
def p_chars(p):
'''chars : CHAR
| chars CHAR'''
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = p[1]
p[0].append(p[2])
pttrns = [
'abc',
'(abc)',
'(abc|def)',
'abc(def|ghi)(jkl|mno)pqr',
'a(b|(c|d))',
]
parser = yacc.yacc()
for pttrn in pttrns:
print('====')
print('pttrn: %s' % (pttrn))
result = parser.parse(pttrn)
print('parsed: %s' % (result))
====
pttrn: abc
parsed: ['a', 'b', 'c']
====
pttrn: (abc)
parsed: [{'or': [['a', 'b', 'c']]}]
====
pttrn: (abc|def)
parsed: [{'or': [['a', 'b', 'c'], 'd', 'e', 'f']}]
====
pttrn: abc(def|ghi)(jkl|mno)pqr
parsed: ['a', 'b', 'c', {'or': [['d', 'e', 'f'], 'g', 'h', 'i']}, {'or': [['j', 'k', 'l'], 'm', 'n', 'o']}, 'p', 'q', 'r']
====
pttrn: a(b|(c|d))
parsed: None
