模型一：
Github: ConvKB
论文: A Novel Embedding Model for Knowledge Base CompletionBased on Convolutional Neural Network
模型二：
GitHub: CapsE
论文: A Capsule Network-based Embedding Model for Knowledge Graph Completion and Search Personalization
注：CapsE模型是在ConvKB模型的启发之下建立的。
ConvKB结构
CapsE结构
model.py (ConvKB)
import tensorflow as tf
import numpy as np
import math
class ConvKB(object):
def __init__(self, sequence_length, num_classes, embedding_size, filter_sizes, num_filters, vocab_size,
pre_trained=[], l2_reg_lambda=0.001, is_trainable=True, useConstantInit=False):
# Placeholders for input, output and dropout
# 占位符
self.input_x = tf.placeholder(tf.int32, [None, sequence_length], name="input_x") #shape=(128,3)
self.input_y = tf.placeholder(tf.float32, [None, num_classes], name="input_y") #shape=(128,1)
self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")
# Keeping track of l2 regularization loss (optional)
# 初始化L2正则化损失值为常数0.0
l2_loss = tf.constant(0.0)
# Embedding layer
# 嵌入层，可以是随机初始化的，也可以是提前训练好的(预训练的)
# self.W是embedding，self.embedded_chars是由input_x映射出的矩阵，self.embedded_chars_expanded是扩充了一个维度，方便使用2D卷积
with tf.name_scope("embedding"):
if pre_trained == []:
self.W = tf.Variable(tf.random_uniform([vocab_size, embedding_size], -math.sqrt(1.0/embedding_size), math.sqrt(1.0/embedding_size), seed=1234), name="W")
else:
self.W = tf.get_variable(name="W2", initializer=pre_trained) #trainable=is_trainable)
self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x) #shape=(128,3,50)
self.embedded_chars_expanded = tf.expand_dims(self.embedded_chars, -1) #shape=(128,3,50,1)
# Create a convolution + maxpool layer for each filter size
# 针对每个过滤器创建卷积操作+最大池化操作
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
with tf.name_scope("conv-maxpool-%s" % filter_size):
if useConstantInit == False:
filter_shape = [sequence_length, filter_size, 1, num_filters] #shape=(3,1,1,500)
W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1, seed=1234), name="W") #shape=(3,1,1,500)
else:
init1 = tf.constant([[[[0.1]]], [[[0.1]]], [[[-0.1]]]])
weight_init = tf.tile(init1, [1, filter_size, 1, num_filters])
W = tf.get_variable(name="W3", initializer=weight_init)
b = tf.Variable(tf.constant(0.0, shape=[num_filters]), name="b")
conv = tf.nn.conv2d(
self.embedded_chars_expanded,
W,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu") #shape=(128,1,50,500)
pooled_outputs.append(h)
# Combine all the pooled features
# 将每一层的池化的特征进行拼接，然后扁平化处理
self.h_pool = tf.concat(pooled_outputs, 2) #shape=(128,1,50,500)
total_dims = (embedding_size * len(filter_sizes) - sum(filter_sizes) + len(filter_sizes)) * num_filters
self.h_pool_flat = tf.reshape(self.h_pool, [-1, total_dims]) #shape=(128,25000)
# Add dropout
with tf.name_scope("dropout"):
self.h_drop = tf.nn.dropout(self.h_pool_flat, self.dropout_keep_prob)
# Final (unnormalized) scores and predictions
# 将以上的特征进行归一化，打分
with tf.name_scope("output"):
W = tf.get_variable(
"W",
shape=[total_dims, num_classes],
initializer=tf.contrib.layers.xavier_initializer(seed=1234)) #shape=(25000,1)
b = tf.Variable(tf.constant(0.0, shape=[num_classes]), name="b")
l2_loss += tf.nn.l2_loss(W)
l2_loss += tf.nn.l2_loss(b)
self.scores = tf.nn.xw_plus_b(self.h_drop, W, b, name="scores")
self.predictions = tf.nn.sigmoid(self.scores) #shape=(128,1)
# Calculate loss
with tf.name_scope("loss"):
losses = tf.nn.softplus(self.scores * self.input_y)
self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=500)
capsuleNet.py (CapsE)
import tensorflow as tf
from capsuleLayer import CapsLayer
import math
epsilon = 1e-9
class CapsE(object):
def __init__(self, sequence_length, embedding_size, num_filters, vocab_size, iter_routing, batch_size=256,
num_outputs_secondCaps=1, vec_len_secondCaps=10, initialization=[], filter_size=1, useConstantInit=False):
# Placeholders for input, output
# 占位符
self.input_x = tf.placeholder(tf.int32, [batch_size, sequence_length], name="input_x") #shape=(256,3)
self.input_y = tf.placeholder(tf.float32, [batch_size, 1], name="input_y") #shape=(256,1)
self.filter_size = filter_size #1
self.num_filters = num_filters #400
self.sequence_length = sequence_length #3
self.embedding_size = embedding_size #100
self.iter_routing = iter_routing #1
self.num_outputs_secondCaps = num_outputs_secondCaps #1
self.vec_len_secondCaps = vec_len_secondCaps #10
self.batch_size = batch_size #256
self.useConstantInit = useConstantInit #false
# Embedding layer
# 嵌入层，可以是随机初始化的，也可以是提前训练好的(预训练的)
# self.W是embedding，self.embedded_chars是由input_x映射出的矩阵，self.embedded_chars_expanded是扩充了一个维度，方便使用2D卷积
with tf.name_scope("embedding"):
if initialization == []:
self.W = tf.Variable(
tf.random_uniform([vocab_size, embedding_size], -math.sqrt(1.0 / embedding_size),
math.sqrt(1.0 / embedding_size), seed=1234), name="W")
else:
self.W = tf.get_variable(name="W2", initializer=initialization) #shape=(40954,100)
self.embedded_chars = tf.nn.embedding_lookup(self.W, self.input_x) #shape=(256,3,100)
self.X = tf.expand_dims(self.embedded_chars, -1) #shape=(256,3,100,1)
self.build_arch() ##构建两层胶囊网络层
self.loss()
self.saver = tf.train.Saver(tf.global_variables(), max_to_keep=500)
tf.logging.info('Seting up the main structure')
def build_arch(self):
#The first capsule layer
#第一层胶囊网络层
with tf.variable_scope('FirstCaps_layer'):
self.firstCaps = CapsLayer(num_outputs_secondCaps=self.num_outputs_secondCaps, vec_len_secondCaps=self.vec_len_secondCaps,
with_routing=False, layer_type='CONV', embedding_size=self.embedding_size,
batch_size=self.batch_size, iter_routing=self.iter_routing,
useConstantInit=self.useConstantInit, filter_size=self.filter_size,
num_filters=self.num_filters, sequence_length=self.sequence_length)
self.caps1 = self.firstCaps(self.X, kernel_size=1, stride=1) #shape=(256,100,400,1)
#The second capsule layer
#第二层胶囊网络层
with tf.variable_scope('SecondCaps_layer'):
self.secondCaps = CapsLayer(num_outputs_secondCaps=self.num_outputs_secondCaps, vec_len_secondCaps=self.vec_len_secondCaps,
with_routing=True, layer_type='FC',
batch_size=self.batch_size, iter_routing=self.iter_routing,
embedding_size=self.embedding_size, useConstantInit=self.useConstantInit, filter_size=self.filter_size,
num_filters=self.num_filters, sequence_length=self.sequence_length)
self.caps2 = self.secondCaps(self.caps1) #shape=(256,1,10,1)
self.v_length = tf.sqrt(tf.reduce_sum(tf.square(self.caps2), axis=2, keep_dims=True) + epsilon) #shape=(256,1,1,1)
def loss(self):
self.scores = tf.reshape(self.v_length, [self.batch_size, 1]) #shape=(256,1)
self.predictions = tf.nn.sigmoid(self.scores)
print("Using square softplus loss")
losses = tf.square(tf.nn.softplus(self.scores * self.input_y))
self.total_loss = tf.reduce_mean(losses)
网友评论