Skip to content

my flutter sign language detector is not detecting anything #554

@janrabang

Description

@janrabang

i am using SSDMOBILENETV2 FPN LITE 320x32 pretrained model and i trained it with my custom sign language datasets.

here is my object_detection_service.dart
`import 'package:camera/camera.dart';
import 'package:tflite_flutter/tflite_flutter.dart';
import 'package:image/image.dart' as img;
import 'dart:developer' as dev;

class InferenceData {
final Interpreter interpreter;
final List<List<List<List>>> inputData;

InferenceData(this.interpreter, this.inputData);
}

class InferenceResult {
final List scores;
final List<List> boxes;
final List classes;

InferenceResult(this.scores, this.boxes, this.classes);
}

class ObjectDetectionService {
static const int inputSize = 320;
static const double confidenceThreshold = 0.1;

static void _log(String message) {
dev.log('[ObjectDetection] $message');
}

static Future<List<List<List<List>>>> preprocessImageIsolate(CameraImage image) async {
try {
final img.Image rgbImage;
if (image.format.group == ImageFormatGroup.yuv420) {
rgbImage = _convertYUV420(image);
} else if (image.format.group == ImageFormatGroup.bgra8888) {
rgbImage = _convertBGRA8888(image);
} else {
throw Exception('Unsupported image format: ${image.format.group}');
}

  final resized = img.copyResize(
    rgbImage,
    width: inputSize,
    height: inputSize,
    interpolation: img.Interpolation.linear,
  );

  final input = List.generate(
    1,
    (index) => List.generate(
      inputSize,
      (y) => List.generate(
        inputSize,
        (x) => List.generate(
          3,
          (c) {
            final pixel = resized.getPixel(x, y);
            // Normalize to [0, 1] instead of [-1, 1]
            return c == 0
                ? pixel.r / 255.0
                : c == 1
                    ? pixel.g / 255.0
                    : pixel.b / 255.0;
          },
        ),
      ),
    ),
  );

  _log('Image preprocessed');
  return input;
} catch (e, stack) {
  _log('Preprocessing error: $e\n$stack');
  throw Exception('Preprocessing failed: $e');
}

}

static img.Image _convertYUV420(CameraImage image) {
final width = image.width;
final height = image.height;
final yPlane = image.planes[0].bytes;
final uPlane = image.planes[1].bytes;
final vPlane = image.planes[2].bytes;

final yRowStride = image.planes[0].bytesPerRow;
final uvRowStride = image.planes[1].bytesPerRow;
final uvPixelStride = image.planes[1].bytesPerPixel!;

final output = img.Image(width: width, height: height);

for (int y = 0; y < height; y++) {
  for (int x = 0; x < width; x++) {
    final int yIndex = y * yRowStride + x;
    final int uvIndex = (y ~/ 2) * uvRowStride + (x ~/ 2) * uvPixelStride;

    final yValue = yPlane[yIndex];
    final uValue = uPlane[uvIndex];
    final vValue = vPlane[uvIndex];

    // Using standard YUV to RGB conversion
    final int r = (yValue + (1.370705 * (vValue - 128))).toInt().clamp(0, 255);
    final int g = (yValue - (0.698001 * (vValue - 128)) - (0.337633 * (uValue - 128))).toInt().clamp(0, 255);
    final int b = (yValue + (1.732446 * (uValue - 128))).toInt().clamp(0, 255);

    output.setPixelRgb(x, y, r, g, b);
  }
}

return output;

}

static img.Image _convertBGRA8888(CameraImage image) {
return img.Image.fromBytes(
width: image.width,
height: image.height,
bytes: image.planes[0].bytes.buffer,
order: img.ChannelOrder.bgra,
);
}

static Future<InferenceResult?> runInferenceIsolate(InferenceData data) async {
try {
final outputBoxes = List<List>.generate(
100,
(_) => List.filled(4, 0.0),
);
final outputClasses = List.filled(100, 0);
final outputScores = List.filled(100, 0);
final outputCount = [1.0];

  final outputs = {
    0: outputScores,
    1: outputBoxes,
    2: outputCount,
    3: outputClasses,
  };

  data.interpreter.runForMultipleInputs([data.inputData], outputs);

  // Debug logging
  _log('Scores: ${outputScores.take(5).toList()}');
  _log('Classes: ${outputClasses.take(5).toList()}');
  _log('First box: ${outputBoxes[0]}');

  return InferenceResult(
    outputScores,
    outputBoxes,
    outputClasses,
  );
} catch (e, stack) {
  _log('Inference error: $e\n$stack');
  return null;
}

}
}

`

and my scan_controller.dart:
`import 'dart:developer' as dev;
import 'package:camera/camera.dart';
import 'package:get/get.dart';
import 'package:tflite_flutter/tflite_flutter.dart';
import 'package:flutter/services.dart';
import 'package:flutter/foundation.dart';
import 'camera_service.dart';
import 'object_detection_service.dart';

class ScanController extends GetxController {
final List cameras;
late CameraService cameraService;
late ObjectDetectionService detectionService;
Interpreter? interpreter;
List labels = [];

RxBool isCameraInitialized = false.obs;
RxString errorMessage = ''.obs;
RxString label = ''.obs;
RxDouble x = 0.0.obs;
RxDouble y = 0.0.obs;
RxDouble w = 0.0.obs;
RxDouble h = 0.0.obs;
RxBool isProcessing = false.obs;
RxBool isTabActive = true.obs;

static const String modelPath = 'assets/model.tflite';
static const String labelsPath = 'assets/labels.txt';
static const Duration _processingInterval = Duration(milliseconds: 100);
DateTime _lastProcessingTime = DateTime.now();

ScanController({required this.cameras}) {
cameraService = CameraService(cameras: cameras);
detectionService = ObjectDetectionService();
}

void _log(String message) {
dev.log('[ScanController] $message');
}

@OverRide
void onInit() {
super.onInit();
_initialize();
}

@OverRide
void onClose() {
disposeResources();
super.onClose();
}

Future _initialize() async {
try {
_log('Starting initialization');

  final options = InterpreterOptions()..threads = 4;
  
  interpreter = await Interpreter.fromAsset(
    modelPath,
    options: options,
  );

  // Log interpreter details
  final inputTensor = interpreter!.getInputTensor(0);
  final outputTensor = interpreter!.getOutputTensor(0);
  _log('Input tensor shape: ${inputTensor.shape}');
  _log('Output tensor shape: ${outputTensor.shape}');

  await loadLabels();
  await initializeCamera();
  _log('Initialization complete');
} catch (e, stack) {
  errorMessage.value = 'Initialization error: $e';
  _log('Initialization error: $e\n$stack');
}

}

Future loadLabels() async {
try {
final labelData = await rootBundle.loadString(labelsPath);
labels = labelData.split('\n')
.where((label) => label.trim().isNotEmpty)
.toList();
_log('Labels loaded: ${labels.length}');
_log('First 5 labels: ${labels.take(5).toList()}');
} catch (e) {
_log('Error loading labels: $e');
rethrow;
}
}

Future initializeCamera() async {
try {
await cameraService.initialize();
if (isTabActive.value) {
await startCamera();
}
isCameraInitialized.value = true;
} catch (e) {
errorMessage.value = e.toString();
_log('Camera initialization error: $e');
rethrow;
}
}

Future startCamera() async {
if (!isCameraInitialized.value) return;
await cameraService.startImageStream(_processCameraImage);
}

Future stopCamera() async {
await cameraService.stopImageStream();
}

Future disposeResources() async {
try {
await cameraService.dispose();
interpreter?.close();
isProcessing.value = false;
isCameraInitialized.value = false;
} catch (e) {
_log('Error during resource disposal: $e');
}
}

Future _processCameraImage(CameraImage image) async {
if (isProcessing.value) return;

final now = DateTime.now();
if (now.difference(_lastProcessingTime) < _processingInterval) return;

_lastProcessingTime = now;
isProcessing.value = true;

try {
  if (interpreter == null) {
    _log('Interpreter not ready');
    return;
  }

  final inputData = await compute(
    ObjectDetectionService.preprocessImageIsolate,
    image,
  );
  _log('Image preprocessed');

  final outputs = await compute(
    ObjectDetectionService.runInferenceIsolate,
    InferenceData(interpreter!, inputData),
  );
  _log('Inference run completed');

  if (outputs != null) {
    _processDetections(
      outputs.scores,
      outputs.boxes,
      outputs.classes,
    );
  }
} catch (e, stack) {
  _log('Processing error: $e\n$stack');
} finally {
  isProcessing.value = false;
}

}

void _processDetections(
List scores,
List<List> boxes,
List classes,
) {
try {
double maxScore = 0;
int maxIndex = -1;

  _log('Processing detections:');
  _log('Scores: ${scores.take(5)}');
  _log('Classes: ${classes.take(5)}');

  for (var i = 0; i < scores.length; i++) {
    if (scores[i] > maxScore && 
        scores[i] > ObjectDetectionService.confidenceThreshold) {
      maxScore = scores[i];
      maxIndex = i;
    }
  }

  if (maxIndex != -1) {
    final box = boxes[maxIndex];
    final classIndex = classes[maxIndex].toInt();
    if (classIndex < labels.length) {
      label.value = '${labels[classIndex]} ${(maxScore * 100).toStringAsFixed(0)}%';
      y.value = box[0];
      x.value = box[1];
      h.value = box[2] - box[0];
      w.value = box[3] - box[1];
      _log('Detection: ${label.value} at ($x, $y) with size ($w, $h)');
    }
  } else {
    label.value = '';
  }
} catch (e, stack) {
  _log('Detection processing error: $e\n$stack');
}

}
}`

i have been working on this for weeks now and i dont know what else to do, i am also new to programming.

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions