fix automorphism_group_generators for indefinite, ambiguous, isotropic, binary quadratic forms

src/QuadForm/Quad/GenusRep.jl

@@ -2254,24 +2254,39 @@ function automorphism_group_generators(g::QuadBin{ZZRingElem})
   d = discriminant(g)
   @assert d > 0
   if is_square(d)
+    # if d is a square, the form represents zero
+    # Let e_1, e_2 be primitive with e_1^2 = 0, e_2^2 = 0 and e_1.e_2>0.
+    # Then any isometry preserves the set {e_1,e_2, -e_1, -e_2}.
+    # We see that the orthogonal group is generated by -id and
+    # possibly the one exchanging e_1 <-> e_2 if it is integral.
+    push!(gens, matrix(FlintZZ, 2, 2, [-1, 0, 0, -1]))
+
     g = primitive_form(g)
+    gg = binary_quadratic_form(g.a, -g.b, g.c)
+    is_ambiguous = is_equivalent(g, gg, proper = true)
+
     gred, t = reduction_with_transformation(g)
-    push!(gens, matrix(FlintZZ, 2, 2, [-1, 0, 0, -1]))
     a = gred.a
     b = gred.b
     c = gred.c
     @assert a == 0 || c == 0
-    if a == c == 0
-      push!(gens, t * matrix(FlintZZ, 2, 2, [0, 1, 1, 0]) * inv(t))
-    elseif a == 0 && c != 0
+
+    # bring it to the form x^2 + b xy
+    if a == 0 && c != 0
       a = gred.c
       c = gred.a
       t = t * matrix(ZZ, 2, 2, [0, 1, 1, 0])
-    elseif a != 0 && c ==0 && b % (2*a) == 0
-      n = b//(2*a)
-      t = t * matrix(ZZ, 2, 2, [1, -n, 0, 1])
-      push!(gens, t * matrix(FlintZZ, 2, 2, [1,0,0,-1]) * inv(t) )
     end
+
+    fl, n = divides(1 - a^2, b)
+    @assert fl == is_ambiguous
+    if fl
+      f = matrix(ZZ, 2, 2, [a, b, n, -a])
+      push!(gens, t* f * inv(t))
+    end
+#     if is_ambiguous && !(a==0 ||(a != 0 && c ==0 && b % (2*a) == 0))
+#       error("missing case")
+#    end
     for T in gens
       @assert _action(g, T) == g
     end

test/QuadForm/QuadBin.jl

@@ -216,7 +216,7 @@
 
     g = binary_quadratic_form(-4, 3, 0)
     gens = automorphism_group_generators(g)
-    @test gens == [ZZ[-1 0;0 -1]]
+    @test gens == [ZZ[-1 0;0 -1], ZZ[-4 3; -5 4]]
 
     g = binary_quadratic_form(1, 2, 0)
     gens = automorphism_group_generators(g)