From 48f97c434815b024b4f29f66f6801be1c6f4739a Mon Sep 17 00:00:00 2001 From: muaddibusulll Date: Sun, 16 Jun 2024 02:06:30 +0530 Subject: [PATCH 1/3] first commit --- mobula/layers/BatchNorm.py | 33 +++++++-------------- mobula/layers/ContrastiveLoss.py | 6 ++-- mobula/layers/CrossEntropy.py | 12 +++++--- mobula/layers/MSE.py | 6 +++- mobula/layers/Tanh.py | 6 +++- requirements.txt | 2 ++ setup.py | 50 ++++++++++++++++---------------- 7 files changed, 59 insertions(+), 56 deletions(-) create mode 100644 requirements.txt diff --git a/mobula/layers/BatchNorm.py b/mobula/layers/BatchNorm.py index 871dd0a..a441eb1 100644 --- a/mobula/layers/BatchNorm.py +++ b/mobula/layers/BatchNorm.py @@ -1,12 +1,14 @@ from .Layer import * +import numpy as np class BatchNorm(Layer): def __init__(self, model, *args, **kwargs): - Layer.__init__(self, model, *args, **kwargs) + super().__init__(model, *args, **kwargs) self.momentum = kwargs.get('momentum', 0.9) self.eps = kwargs.get('eps', 1e-5) self.use_global_stats = kwargs.get('use_global_stats', False) self.axis = kwargs.get('axis', 1) + def reshape(self): assert 0 <= self.axis < self.X.ndim self.cshape = [1] * self.X.ndim @@ -14,43 +16,28 @@ def reshape(self): self.valid_axes = tuple([i for i in range(self.X.ndim) if i != self.axis]) self.Y = np.zeros(self.X.shape) - # (1, C, 1, 1) self.W = np.ones(self.cshape) - # (1, C, 1, 1) self.b = np.zeros(self.cshape) - # Current Mean self.moving_mean = np.zeros(self.cshape) self.moving_var = np.ones(self.cshape) + def forward(self): if self.is_training() and not self.use_global_stats: - # The mean and var of this batch - self.batch_mean = np.mean(self.X, self.valid_axes, keepdims = True) - self.batch_var = np.mean(np.square(self.X - self.batch_mean), self.valid_axes, keepdims = True) # Is it faster than np.var? - # compute moving mean and moving var + self.batch_mean = np.mean(self.X, self.valid_axes, keepdims=True) + self.batch_var = np.mean(np.square(self.X - self.batch_mean), self.valid_axes, keepdims=True) self.moving_mean = self.moving_mean * self.momentum + self.batch_mean * (1 - self.momentum) self.moving_var = self.moving_var * self.momentum + self.batch_var * (1 - self.momentum) else: self.batch_mean = self.moving_mean self.batch_var = self.moving_var - # Normalize - # [TODO] when eps == 0 ? + self.nd = 1.0 / np.sqrt(self.batch_var + self.eps) self.nx = (self.X - self.batch_mean) * self.nd - # Scale and Shift self.Y = np.multiply(self.nx, self.W) + self.b + def backward(self): - # Compute self.dX, self.dW, self.db dnx = np.multiply(self.dY, self.W) xsm = self.X - self.batch_mean m = self.X.shape[0] - dvar = np.sum(np.multiply(dnx, xsm), self.valid_axes, keepdims = True) * (-0.5) * np.power(self.nd, 3.0) - dmean = -self.nd * np.sum(dnx, self.valid_axes, keepdims = True) - dvar * np.mean(xsm, self.valid_axes, keepdims = True) * 2.0 - self.dX = dnx * self.nd + dvar * xsm * (2.0 / m) + dmean * (1.0 / m) - self.dW = np.sum(self.dY * self.nx, self.valid_axes, keepdims = True) - self.db = np.sum(self.dY, self.valid_axes, keepdims = True) - @property - def params(self): - return [self.W, self.b, self.moving_mean, self.moving_var] - @property - def grads(self): - return [self.dW, self.db] + dvar = np.sum(np.multiply(dnx, xsm), self.valid_axes, keepdims=True) * (-0.5) * np.power(self.nd, 3.0) + dmean = -self.nd \ No newline at end of file diff --git a/mobula/layers/ContrastiveLoss.py b/mobula/layers/ContrastiveLoss.py index 058a7b2..2dc49f8 100644 --- a/mobula/layers/ContrastiveLoss.py +++ b/mobula/layers/ContrastiveLoss.py @@ -1,10 +1,11 @@ from .LossLayer import * +import numpy as np class ContrastiveLoss(LossLayer): def __init__(self, models, *args, **kwargs): - # models = [X1, X2, sim] - LossLayer.__init__(self, models, *args, **kwargs) + super().__init__(models, *args, **kwargs) self.margin = kwargs.get("margin", 1.0) + def forward(self): self.sim = (self.X[2] == 1).ravel() n = self.sim.shape[0] @@ -14,6 +15,7 @@ def forward(self): df = (self.margin - self.dist).ravel() self.bdf = ((~self.sim) & (df > 0)) self.Y = (np.sum(dist_sq[self.sim]) + np.sum(np.square(df[self.bdf]))) / (2.0 * n) + def backward(self): n = self.sim.shape[0] dX = np.zeros(self.X[0].shape) diff --git a/mobula/layers/CrossEntropy.py b/mobula/layers/CrossEntropy.py index 9e99c28..7a6dd78 100644 --- a/mobula/layers/CrossEntropy.py +++ b/mobula/layers/CrossEntropy.py @@ -1,12 +1,16 @@ from .LossLayer import * +import numpy as np class CrossEntropy(LossLayer): def __init__(self, model, *args, **kwargs): - LossLayer.__init__(self, model, *args, **kwargs) + super().__init__(model, *args, **kwargs) + def reshape(self): self.Y = 0.0 + def forward(self): - self.Y = np.mean(- np.multiply(self.label, np.log(self.X)) - \ - np.multiply(1.0 - self.label, np.log(1.0 - self.X))) + self.Y = np.mean(- np.multiply(self.label, np.log(self.X + 1e-15)) - \ + np.multiply(1.0 - self.label, np.log(1.0 - self.X + 1e-15))) + def backward(self): - self.dX = (-self.label / self.X + (1.0 - self.label) / (1.0 - self.X)) * self.dY + self.dX = (-self.label / (self.X + 1e-15) + (1.0 - self.label) / (1.0 - self.X + 1e-15)) * self.dY diff --git a/mobula/layers/MSE.py b/mobula/layers/MSE.py index 1cca545..3714d93 100644 --- a/mobula/layers/MSE.py +++ b/mobula/layers/MSE.py @@ -1,12 +1,16 @@ from .LossLayer import * +import numpy as np class MSE(LossLayer): def __init__(self, model, *args, **kwargs): - LossLayer.__init__(self, model, *args, **kwargs) + super().__init__(model, *args, **kwargs) + def reshape(self): self.Y = 0.0 + def forward(self): self.d = (self.X - self.label) self.Y = np.mean(np.square(self.d)) + def backward(self): self.dX = (2 * self.d) * self.dY diff --git a/mobula/layers/Tanh.py b/mobula/layers/Tanh.py index 28f2252..9292abc 100644 --- a/mobula/layers/Tanh.py +++ b/mobula/layers/Tanh.py @@ -1,11 +1,15 @@ from .Layer import * +import numpy as np class Tanh(Layer): def __init__(self, model, *args, **kwargs): - Layer.__init__(self, model, *args, **kwargs) + super().__init__(model, *args, **kwargs) + def reshape(self): self.Y = np.zeros(self.X.shape) + def forward(self): self.Y = 2.0 / (1.0 + np.exp(-2.0 * self.X)) - 1.0 + def backward(self): self.dX = np.multiply(self.dY, 1.0 - np.square(self.Y)) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1750d49 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +numpy +numpy_groupies diff --git a/setup.py b/setup.py index 004dc1e..c1c3eca 100644 --- a/setup.py +++ b/setup.py @@ -1,29 +1,29 @@ from setuptools import setup, find_packages setup( - name = 'mobula', - version = '1.0.1', - description = 'A Lightweight & Flexible Deep Learning (Neural Network) Framework in Python', - author = 'wkcn', - author_email = 'wkcn@live.cn', - url = 'https://github.com/wkcn/mobula', - packages = find_packages(), - package_data = { - '' : ['*.md'], - 'docs' : ['docs/*.md'], - 'examples' : ['examples/*.py'] - }, - keywords = 'Deep Learning Framework in Python', - license = 'MIT', - classifiers = [ - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 3', - 'Topic :: Scientific/Engineering :: Mathematics', - 'License :: OSI Approved :: MIT License' - ], - install_requires = [ - 'numpy', - 'numpy_groupies' - ] + name='mobula', + version='1.0.2', + description='A Lightweight & Flexible Deep Learning (Neural Network) Framework in Python', + author='wkcn', + author_email='wkcn@live.cn', + url='https://github.com/wkcn/mobula', + packages=find_packages(), + package_data={ + '': ['*.md'], + 'docs': ['docs/*.md'], + 'examples': ['examples/*.py'] + }, + keywords='Deep Learning Framework in Python', + license='MIT', + classifiers=[ + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Topic :: Scientific/Engineering :: Mathematics', + 'License :: OSI Approved :: MIT License' + ], + install_requires=[ + 'numpy', + 'numpy_groupies' + ], + python_requires='>=3.6', ) From 475dc819aa45eaf95a7b62ac888df36e7cbeaf3b Mon Sep 17 00:00:00 2001 From: wkcn Date: Wed, 19 Jun 2024 00:01:21 +0800 Subject: [PATCH 2/3] trigger CI From f489b6a271bb968e7fdd4a19458de63698a52ca9 Mon Sep 17 00:00:00 2001 From: wkcn Date: Wed, 19 Jun 2024 00:04:18 +0800 Subject: [PATCH 3/3] fix BatchNorm --- mobula/layers/BatchNorm.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/mobula/layers/BatchNorm.py b/mobula/layers/BatchNorm.py index a441eb1..06b5824 100644 --- a/mobula/layers/BatchNorm.py +++ b/mobula/layers/BatchNorm.py @@ -1,5 +1,4 @@ from .Layer import * -import numpy as np class BatchNorm(Layer): def __init__(self, model, *args, **kwargs): @@ -16,28 +15,47 @@ def reshape(self): self.valid_axes = tuple([i for i in range(self.X.ndim) if i != self.axis]) self.Y = np.zeros(self.X.shape) + # (1, C, 1, 1) self.W = np.ones(self.cshape) + # (1, C, 1, 1) self.b = np.zeros(self.cshape) + # Current Mean self.moving_mean = np.zeros(self.cshape) self.moving_var = np.ones(self.cshape) def forward(self): if self.is_training() and not self.use_global_stats: - self.batch_mean = np.mean(self.X, self.valid_axes, keepdims=True) - self.batch_var = np.mean(np.square(self.X - self.batch_mean), self.valid_axes, keepdims=True) + # The mean and var of this batch + self.batch_mean = np.mean(self.X, self.valid_axes, keepdims = True) + self.batch_var = np.mean(np.square(self.X - self.batch_mean), self.valid_axes, keepdims = True) # Is it faster than np.var? + # compute moving mean and moving var self.moving_mean = self.moving_mean * self.momentum + self.batch_mean * (1 - self.momentum) self.moving_var = self.moving_var * self.momentum + self.batch_var * (1 - self.momentum) else: self.batch_mean = self.moving_mean self.batch_var = self.moving_var - + # Normalize + # [TODO] when eps == 0 ? self.nd = 1.0 / np.sqrt(self.batch_var + self.eps) self.nx = (self.X - self.batch_mean) * self.nd + # Scale and Shift self.Y = np.multiply(self.nx, self.W) + self.b def backward(self): + # Compute self.dX, self.dW, self.db dnx = np.multiply(self.dY, self.W) xsm = self.X - self.batch_mean m = self.X.shape[0] - dvar = np.sum(np.multiply(dnx, xsm), self.valid_axes, keepdims=True) * (-0.5) * np.power(self.nd, 3.0) - dmean = -self.nd \ No newline at end of file + dvar = np.sum(np.multiply(dnx, xsm), self.valid_axes, keepdims = True) * (-0.5) * np.power(self.nd, 3.0) + dmean = -self.nd * np.sum(dnx, self.valid_axes, keepdims = True) - dvar * np.mean(xsm, self.valid_axes, keepdims = True) * 2.0 + self.dX = dnx * self.nd + dvar * xsm * (2.0 / m) + dmean * (1.0 / m) + self.dW = np.sum(self.dY * self.nx, self.valid_axes, keepdims = True) + self.db = np.sum(self.dY, self.valid_axes, keepdims = True) + + @property + def params(self): + return [self.W, self.b, self.moving_mean, self.moving_var] + + @property + def grads(self): + return [self.dW, self.db]