AI: entropy and corelation
This commit is contained in:
mcDandy
@@ -161,6 +161,7 @@ func1:
|
||||
| UPPER LPAREN expr RPAREN # UpperFunc
|
||||
| LOWER LPAREN expr RPAREN # LowerFunc
|
||||
| TRIM LPAREN expr RPAREN # TrimFunc
|
||||
| ENTROPY LPAREN expr RPAREN # EntropyFunc
|
||||
| DILATE LPAREN expr (COMMA expr)? RPAREN # DilateFunc
|
||||
| ERODE LPAREN expr (COMMA expr)? RPAREN # ErodeFunc
|
||||
| MORPH_OPEN LPAREN expr (COMMA expr)? RPAREN # MorphOpenFunc
|
||||
@@ -181,6 +182,7 @@ func2:
|
||||
| COSSIM LPAREN expr COMMA expr RPAREN # CossimFunc
|
||||
| FLIP LPAREN expr COMMA expr RPAREN # FlipFunc
|
||||
| COV LPAREN expr COMMA expr RPAREN # CovFunc
|
||||
| CORR LPAREN expr COMMA expr RPAREN # CorrFunc
|
||||
| APPEND LPAREN expr COMMA expr RPAREN # AppendFunc
|
||||
| GAUSSIAN LPAREN expr COMMA expr (COMMA expr)? RPAREN # GaussianFunc
|
||||
| TOPK_IND LPAREN expr COMMA expr RPAREN # TopkIndFunc
|
||||
@@ -400,6 +402,8 @@ SOFTMIN: 'softmin';
|
||||
UNIQUE: 'unique';
|
||||
FLIP: 'flip';
|
||||
COV: 'cov';
|
||||
CORR: 'corr' | 'correlation';
|
||||
ENTROPY: 'entropy';
|
||||
CROP: 'crop';
|
||||
NONE: 'none'|'None'|'null'|'NULL';
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -137,59 +137,61 @@ SOFTMIN=136
|
||||
UNIQUE=137
|
||||
FLIP=138
|
||||
COV=139
|
||||
CROP=140
|
||||
NONE=141
|
||||
NOISE=142
|
||||
RAND=143
|
||||
CAUCHY=144
|
||||
EXPONENTIAL=145
|
||||
LOGNORMAL=146
|
||||
BERNOULLI=147
|
||||
POISSON=148
|
||||
GAMMADIST=149
|
||||
BETADIST=150
|
||||
LAPLACEDIST=151
|
||||
GUMBELDIST=152
|
||||
WEIBULLDIST=153
|
||||
CHI2DIST=154
|
||||
STUDENTTDIST=155
|
||||
PERLIN=156
|
||||
CELLULAR=157
|
||||
PLASMA=158
|
||||
PLUS=159
|
||||
MINUS=160
|
||||
MULT=161
|
||||
DIV=162
|
||||
MOD=163
|
||||
POW=164
|
||||
LSHIFT=165
|
||||
RSHIFT=166
|
||||
GE=167
|
||||
GT=168
|
||||
LE=169
|
||||
LT=170
|
||||
EQ=171
|
||||
EQUEALS=172
|
||||
NE=173
|
||||
PIPE=174
|
||||
LPAREN=175
|
||||
RPAREN=176
|
||||
COMMA=177
|
||||
SEMICOLON=178
|
||||
ARROW=179
|
||||
LBRACKET=180
|
||||
RBRACKET=181
|
||||
QUESTION=182
|
||||
COLON=183
|
||||
LBRACE=184
|
||||
RBRACE=185
|
||||
NUMBER=186
|
||||
CONSTANT=187
|
||||
STRING=188
|
||||
VARIABLE=189
|
||||
SL_COMMENT=190
|
||||
ML_COMMENT=191
|
||||
WS=192
|
||||
CORR=140
|
||||
ENTROPY=141
|
||||
CROP=142
|
||||
NONE=143
|
||||
NOISE=144
|
||||
RAND=145
|
||||
CAUCHY=146
|
||||
EXPONENTIAL=147
|
||||
LOGNORMAL=148
|
||||
BERNOULLI=149
|
||||
POISSON=150
|
||||
GAMMADIST=151
|
||||
BETADIST=152
|
||||
LAPLACEDIST=153
|
||||
GUMBELDIST=154
|
||||
WEIBULLDIST=155
|
||||
CHI2DIST=156
|
||||
STUDENTTDIST=157
|
||||
PERLIN=158
|
||||
CELLULAR=159
|
||||
PLASMA=160
|
||||
PLUS=161
|
||||
MINUS=162
|
||||
MULT=163
|
||||
DIV=164
|
||||
MOD=165
|
||||
POW=166
|
||||
LSHIFT=167
|
||||
RSHIFT=168
|
||||
GE=169
|
||||
GT=170
|
||||
LE=171
|
||||
LT=172
|
||||
EQ=173
|
||||
EQUEALS=174
|
||||
NE=175
|
||||
PIPE=176
|
||||
LPAREN=177
|
||||
RPAREN=178
|
||||
COMMA=179
|
||||
SEMICOLON=180
|
||||
ARROW=181
|
||||
LBRACKET=182
|
||||
RBRACKET=183
|
||||
QUESTION=184
|
||||
COLON=185
|
||||
LBRACE=186
|
||||
RBRACE=187
|
||||
NUMBER=188
|
||||
CONSTANT=189
|
||||
STRING=190
|
||||
VARIABLE=191
|
||||
SL_COMMENT=192
|
||||
ML_COMMENT=193
|
||||
WS=194
|
||||
'sin'=1
|
||||
'cos'=2
|
||||
'tan'=3
|
||||
@@ -306,31 +308,32 @@ WS=192
|
||||
'unique'=137
|
||||
'flip'=138
|
||||
'cov'=139
|
||||
'crop'=140
|
||||
'+'=159
|
||||
'-'=160
|
||||
'*'=161
|
||||
'/'=162
|
||||
'%'=163
|
||||
'^'=164
|
||||
'<<'=165
|
||||
'>>'=166
|
||||
'>='=167
|
||||
'>'=168
|
||||
'<='=169
|
||||
'<'=170
|
||||
'=='=171
|
||||
'='=172
|
||||
'!='=173
|
||||
'|'=174
|
||||
'('=175
|
||||
')'=176
|
||||
','=177
|
||||
';'=178
|
||||
'->'=179
|
||||
'['=180
|
||||
']'=181
|
||||
'?'=182
|
||||
':'=183
|
||||
'{'=184
|
||||
'}'=185
|
||||
'entropy'=141
|
||||
'crop'=142
|
||||
'+'=161
|
||||
'-'=162
|
||||
'*'=163
|
||||
'/'=164
|
||||
'%'=165
|
||||
'^'=166
|
||||
'<<'=167
|
||||
'>>'=168
|
||||
'>='=169
|
||||
'>'=170
|
||||
'<='=171
|
||||
'<'=172
|
||||
'=='=173
|
||||
'='=174
|
||||
'!='=175
|
||||
'|'=176
|
||||
'('=177
|
||||
')'=178
|
||||
','=179
|
||||
';'=180
|
||||
'->'=181
|
||||
'['=182
|
||||
']'=183
|
||||
'?'=184
|
||||
':'=185
|
||||
'{'=186
|
||||
'}'=187
|
||||
|
||||
File diff suppressed because one or more lines are too long
+866
-851
File diff suppressed because it is too large
Load Diff
@@ -137,59 +137,61 @@ SOFTMIN=136
|
||||
UNIQUE=137
|
||||
FLIP=138
|
||||
COV=139
|
||||
CROP=140
|
||||
NONE=141
|
||||
NOISE=142
|
||||
RAND=143
|
||||
CAUCHY=144
|
||||
EXPONENTIAL=145
|
||||
LOGNORMAL=146
|
||||
BERNOULLI=147
|
||||
POISSON=148
|
||||
GAMMADIST=149
|
||||
BETADIST=150
|
||||
LAPLACEDIST=151
|
||||
GUMBELDIST=152
|
||||
WEIBULLDIST=153
|
||||
CHI2DIST=154
|
||||
STUDENTTDIST=155
|
||||
PERLIN=156
|
||||
CELLULAR=157
|
||||
PLASMA=158
|
||||
PLUS=159
|
||||
MINUS=160
|
||||
MULT=161
|
||||
DIV=162
|
||||
MOD=163
|
||||
POW=164
|
||||
LSHIFT=165
|
||||
RSHIFT=166
|
||||
GE=167
|
||||
GT=168
|
||||
LE=169
|
||||
LT=170
|
||||
EQ=171
|
||||
EQUEALS=172
|
||||
NE=173
|
||||
PIPE=174
|
||||
LPAREN=175
|
||||
RPAREN=176
|
||||
COMMA=177
|
||||
SEMICOLON=178
|
||||
ARROW=179
|
||||
LBRACKET=180
|
||||
RBRACKET=181
|
||||
QUESTION=182
|
||||
COLON=183
|
||||
LBRACE=184
|
||||
RBRACE=185
|
||||
NUMBER=186
|
||||
CONSTANT=187
|
||||
STRING=188
|
||||
VARIABLE=189
|
||||
SL_COMMENT=190
|
||||
ML_COMMENT=191
|
||||
WS=192
|
||||
CORR=140
|
||||
ENTROPY=141
|
||||
CROP=142
|
||||
NONE=143
|
||||
NOISE=144
|
||||
RAND=145
|
||||
CAUCHY=146
|
||||
EXPONENTIAL=147
|
||||
LOGNORMAL=148
|
||||
BERNOULLI=149
|
||||
POISSON=150
|
||||
GAMMADIST=151
|
||||
BETADIST=152
|
||||
LAPLACEDIST=153
|
||||
GUMBELDIST=154
|
||||
WEIBULLDIST=155
|
||||
CHI2DIST=156
|
||||
STUDENTTDIST=157
|
||||
PERLIN=158
|
||||
CELLULAR=159
|
||||
PLASMA=160
|
||||
PLUS=161
|
||||
MINUS=162
|
||||
MULT=163
|
||||
DIV=164
|
||||
MOD=165
|
||||
POW=166
|
||||
LSHIFT=167
|
||||
RSHIFT=168
|
||||
GE=169
|
||||
GT=170
|
||||
LE=171
|
||||
LT=172
|
||||
EQ=173
|
||||
EQUEALS=174
|
||||
NE=175
|
||||
PIPE=176
|
||||
LPAREN=177
|
||||
RPAREN=178
|
||||
COMMA=179
|
||||
SEMICOLON=180
|
||||
ARROW=181
|
||||
LBRACKET=182
|
||||
RBRACKET=183
|
||||
QUESTION=184
|
||||
COLON=185
|
||||
LBRACE=186
|
||||
RBRACE=187
|
||||
NUMBER=188
|
||||
CONSTANT=189
|
||||
STRING=190
|
||||
VARIABLE=191
|
||||
SL_COMMENT=192
|
||||
ML_COMMENT=193
|
||||
WS=194
|
||||
'sin'=1
|
||||
'cos'=2
|
||||
'tan'=3
|
||||
@@ -306,31 +308,32 @@ WS=192
|
||||
'unique'=137
|
||||
'flip'=138
|
||||
'cov'=139
|
||||
'crop'=140
|
||||
'+'=159
|
||||
'-'=160
|
||||
'*'=161
|
||||
'/'=162
|
||||
'%'=163
|
||||
'^'=164
|
||||
'<<'=165
|
||||
'>>'=166
|
||||
'>='=167
|
||||
'>'=168
|
||||
'<='=169
|
||||
'<'=170
|
||||
'=='=171
|
||||
'='=172
|
||||
'!='=173
|
||||
'|'=174
|
||||
'('=175
|
||||
')'=176
|
||||
','=177
|
||||
';'=178
|
||||
'->'=179
|
||||
'['=180
|
||||
']'=181
|
||||
'?'=182
|
||||
':'=183
|
||||
'{'=184
|
||||
'}'=185
|
||||
'entropy'=141
|
||||
'crop'=142
|
||||
'+'=161
|
||||
'-'=162
|
||||
'*'=163
|
||||
'/'=164
|
||||
'%'=165
|
||||
'^'=166
|
||||
'<<'=167
|
||||
'>>'=168
|
||||
'>='=169
|
||||
'>'=170
|
||||
'<='=171
|
||||
'<'=172
|
||||
'=='=173
|
||||
'='=174
|
||||
'!='=175
|
||||
'|'=176
|
||||
'('=177
|
||||
')'=178
|
||||
','=179
|
||||
';'=180
|
||||
'->'=181
|
||||
'['=182
|
||||
']'=183
|
||||
'?'=184
|
||||
':'=185
|
||||
'{'=186
|
||||
'}'=187
|
||||
|
||||
+1593
-1505
File diff suppressed because it is too large
Load Diff
@@ -664,6 +664,11 @@ class MathExprVisitor(ParseTreeVisitor):
|
||||
return self.visitChildren(ctx)
|
||||
|
||||
|
||||
# Visit a parse tree produced by MathExprParser#EntropyFunc.
|
||||
def visitEntropyFunc(self, ctx:MathExprParser.EntropyFuncContext):
|
||||
return self.visitChildren(ctx)
|
||||
|
||||
|
||||
# Visit a parse tree produced by MathExprParser#DilateFunc.
|
||||
def visitDilateFunc(self, ctx:MathExprParser.DilateFuncContext):
|
||||
return self.visitChildren(ctx)
|
||||
@@ -754,6 +759,11 @@ class MathExprVisitor(ParseTreeVisitor):
|
||||
return self.visitChildren(ctx)
|
||||
|
||||
|
||||
# Visit a parse tree produced by MathExprParser#CorrFunc.
|
||||
def visitCorrFunc(self, ctx:MathExprParser.CorrFuncContext):
|
||||
return self.visitChildren(ctx)
|
||||
|
||||
|
||||
# Visit a parse tree produced by MathExprParser#AppendFunc.
|
||||
def visitAppendFunc(self, ctx:MathExprParser.AppendFuncContext):
|
||||
return self.visitChildren(ctx)
|
||||
|
||||
@@ -1026,7 +1026,7 @@ class UnifiedMathVisitor(MathExprVisitor):
|
||||
target_numel = 1
|
||||
for dim in new_shape:
|
||||
target_numel *= dim
|
||||
|
||||
|
||||
if original_numel != target_numel:
|
||||
raise ValueError(
|
||||
f"{ctx.start.line}:{ctx.start.column}: Cannot reshape tensor of size {original_numel} "
|
||||
@@ -3298,6 +3298,20 @@ class UnifiedMathVisitor(MathExprVisitor):
|
||||
else:
|
||||
return [r, g, b]
|
||||
|
||||
def visitEntropyFunc(self, ctx):
|
||||
val = self._promote_to_tensor((yield ctx.expr()))
|
||||
# Shannon entropy: -sum(p * log(p))
|
||||
p = F.softmax(val.flatten().float(), dim=0)
|
||||
entropy = -torch.sum(p * torch.log(p + 1e-10))
|
||||
return float(entropy.item())
|
||||
|
||||
def visitCorrFunc(self, ctx):
|
||||
x = self._promote_to_tensor((yield ctx.expr(0))).float()
|
||||
y = self._promote_to_tensor((yield ctx.expr(1))).float()
|
||||
# Pearson correlation coefficient
|
||||
vx = x - torch.mean(x)
|
||||
vy = y - torch.mean(y)
|
||||
corr = torch.sum(vx * vy) / (torch.sqrt(torch.sum(vx ** 2)) * torch.sqrt(torch.sum(vy ** 2)))
|
||||
return float(corr.item())
|
||||
|
||||
|
||||
|
||||
@@ -472,6 +472,121 @@ def test_new_loop_features():
|
||||
res6 = parse_and_visit("crop(T, [1, 1], [2, 2])", vars)
|
||||
assert torch.equal(res6, expected_pad)
|
||||
|
||||
def test_entropy():
|
||||
"""Test entropy function for information entropy calculation."""
|
||||
vars = {}
|
||||
|
||||
# 1. Test uniform distribution (maximum entropy)
|
||||
# Uniform probabilities should have high entropy
|
||||
uniform = torch.ones(4) / 4.0 # [0.25, 0.25, 0.25, 0.25]
|
||||
vars["uniform"] = uniform
|
||||
entropy_uniform = parse_and_visit("entropy(uniform)", vars)
|
||||
assert isinstance(entropy_uniform, float)
|
||||
# For uniform distribution of 4 elements: H = -sum(0.25 * log(0.25)) = log(4) ≈ 1.386
|
||||
assert 1.3 < entropy_uniform < 1.5
|
||||
|
||||
# 2. Test deterministic distribution (minimum entropy)
|
||||
# One probability is 1, others are 0 -> entropy should be near 0
|
||||
deterministic = torch.tensor([1000.0, -1000.0, -1000.0, -1000.0]) # After softmax: ~[1, 0, 0, 0]
|
||||
vars["deterministic"] = deterministic
|
||||
entropy_det = parse_and_visit("entropy(deterministic)", vars)
|
||||
assert isinstance(entropy_det, float)
|
||||
assert entropy_det < 0.1 # Near zero entropy
|
||||
|
||||
# 3. Test with different tensor sizes
|
||||
small = torch.randn(8)
|
||||
vars["small"] = small
|
||||
entropy_small = parse_and_visit("entropy(small)", vars)
|
||||
assert isinstance(entropy_small, float)
|
||||
assert entropy_small > 0 # Should be positive
|
||||
|
||||
# 4. Test that entropy is always non-negative
|
||||
random_vals = torch.randn(100)
|
||||
vars["random_vals"] = random_vals
|
||||
entropy_random = parse_and_visit("entropy(random_vals)", vars)
|
||||
assert entropy_random >= 0
|
||||
|
||||
def test_correlation():
|
||||
"""Test correlation (Pearson correlation coefficient) function."""
|
||||
vars = {}
|
||||
|
||||
# 1. Perfect positive correlation
|
||||
x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
y = torch.tensor([2.0, 4.0, 6.0, 8.0, 10.0]) # y = 2*x
|
||||
vars["x"] = x
|
||||
vars["y"] = y
|
||||
corr_perfect = parse_and_visit("corr(x, y)", vars)
|
||||
assert isinstance(corr_perfect, float)
|
||||
assert abs(corr_perfect - 1.0) < 1e-5 # Should be very close to 1
|
||||
|
||||
# Test with alias
|
||||
corr_alias = parse_and_visit("correlation(x, y)", vars)
|
||||
assert abs(corr_alias - 1.0) < 1e-5
|
||||
|
||||
# 2. Perfect negative correlation
|
||||
z = torch.tensor([10.0, 8.0, 6.0, 4.0, 2.0]) # Decreasing
|
||||
vars["z"] = z
|
||||
corr_negative = parse_and_visit("corr(x, z)", vars)
|
||||
assert isinstance(corr_negative, float)
|
||||
assert abs(corr_negative - (-1.0)) < 1e-5 # Should be very close to -1
|
||||
|
||||
# 3. No correlation (orthogonal)
|
||||
a = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
b = torch.tensor([1.0, -1.0, 1.0, -1.0, 1.0]) # Oscillating
|
||||
vars["a"] = a
|
||||
vars["b"] = b
|
||||
corr_none = parse_and_visit("corr(a, b)", vars)
|
||||
assert isinstance(corr_none, float)
|
||||
assert abs(corr_none) < 0.5 # Low correlation
|
||||
|
||||
# 4. Test with same tensor (should be 1.0)
|
||||
corr_self = parse_and_visit("corr(x, x)", vars)
|
||||
assert abs(corr_self - 1.0) < 1e-5
|
||||
|
||||
# 5. Test with flattened 2D tensors
|
||||
t1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
t2 = torch.tensor([[1.5, 3.0], [4.5, 6.0]]) # Scaled version
|
||||
vars["t1"] = t1
|
||||
vars["t2"] = t2
|
||||
corr_2d = parse_and_visit("corr(t1, t2)", vars)
|
||||
assert isinstance(corr_2d, float)
|
||||
assert abs(corr_2d - 1.0) < 1e-5 # Linear relationship
|
||||
|
||||
# 6. Test correlation is symmetric
|
||||
corr_xy = parse_and_visit("corr(x, y)", vars)
|
||||
corr_yx = parse_and_visit("corr(y, x)", vars)
|
||||
assert abs(corr_xy - corr_yx) < 1e-10 # Should be identical
|
||||
|
||||
def test_entropy_and_correlation_edge_cases():
|
||||
"""Test edge cases for entropy and correlation."""
|
||||
vars = {}
|
||||
|
||||
# 1. Entropy with constant values (after softmax becomes uniform)
|
||||
constant = torch.ones(10)
|
||||
vars["constant"] = constant
|
||||
entropy_const = parse_and_visit("entropy(constant)", vars)
|
||||
# All equal logits -> uniform distribution after softmax -> log(10) ≈ 2.302
|
||||
assert 2.2 < entropy_const < 2.4
|
||||
|
||||
# 2. Correlation with constant values (undefined, but should handle gracefully)
|
||||
const_a = torch.ones(5) * 3.0
|
||||
const_b = torch.ones(5) * 5.0
|
||||
vars["const_a"] = const_a
|
||||
vars["const_b"] = const_b
|
||||
# Both have zero variance, correlation is undefined (0/0)
|
||||
# Implementation returns nan or 0
|
||||
corr_const = parse_and_visit("corr(const_a, const_b)", vars)
|
||||
# Check that it doesn't crash and returns a float
|
||||
assert isinstance(corr_const, float)
|
||||
|
||||
# 3. Small tensors
|
||||
tiny_x = torch.tensor([1.0, 2.0])
|
||||
tiny_y = torch.tensor([2.0, 4.0])
|
||||
vars["tiny_x"] = tiny_x
|
||||
vars["tiny_y"] = tiny_y
|
||||
corr_tiny = parse_and_visit("corr(tiny_x, tiny_y)", vars)
|
||||
assert abs(corr_tiny - 1.0) < 1e-5
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
test_scalar_ops()
|
||||
@@ -488,11 +603,15 @@ if __name__ == "__main__":
|
||||
test_pinv()
|
||||
test_pinv()
|
||||
test_quartil()
|
||||
test_percentile()
|
||||
test_custom_functions()
|
||||
test_append()
|
||||
test_random_generators()
|
||||
test_recursion_and_depth()
|
||||
test_new_loop_features()
|
||||
test_entropy()
|
||||
test_correlation()
|
||||
test_entropy_and_correlation_edge_cases()
|
||||
print("All UnifiedMathVisitor tests passed!")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user