1//
2//  EXFaceDetectorManager.m
3//  Exponent
4//
5//  Created by Stanisław Chmiela on 22.11.2017.
6//  Copyright © 2017 650 Industries. All rights reserved.
7
8#import <EXFaceDetector/EXFaceEncoder.h>
9#import <EXFaceDetector/EXFaceDetectorUtils.h>
10#import <EXFaceDetector/EXFaceDetectorModule.h>
11#import <EXFaceDetector/EXFaceDetectorManager.h>
12#import <EXFaceDetector/EXFaceDetector.h>
13#import <EXFaceDetector/EXCSBufferOrientationCalculator.h>
14
15static const NSString *kMinDetectionInterval = @"minDetectionInterval";
16
17@interface EXFaceDetectorManager() <AVCaptureVideoDataOutputSampleBufferDelegate>
18
19@property (assign, nonatomic) long previousFacesCount;
20@property (nonatomic, weak) AVCaptureSession *session;
21@property (nonatomic, assign) BOOL mirroredImageSession;
22@property UIInterfaceOrientation interfaceOrientation;
23@property (nonatomic, weak) dispatch_queue_t sessionQueue;
24@property (nonatomic, copy, nullable) void (^onFacesDetected)(NSArray<NSDictionary *> *);
25@property (nonatomic, weak) AVCaptureVideoPreviewLayer *previewLayer;
26@property (nonatomic, assign, getter=isDetectingFaceEnabled) BOOL faceDetectionEnabled;
27@property (nonatomic, assign, getter=isFaceDetecionRunning) BOOL faceDetectionRunning;
28@property (nonatomic, strong) MLKFaceDetectorOptions* faceDetectorOptions;
29@property (atomic, assign) NSInteger lastFrameCapturedTimeMilis;
30@property (atomic) NSDate *startDetect;
31@property (atomic) BOOL faceDetectionProcessing;
32@property EXFaceDetector *faceDetector;
33@property NSInteger timeIntervalMillis;
34
35@end
36
37@implementation EXFaceDetectorManager
38
39- (instancetype)init
40{
41  return [self initWithOptions:[EXFaceDetectorUtils defaultFaceDetectorOptions]];
42}
43
44- (instancetype)initWithOptions:(NSDictionary*)options
45{
46  if (self = [super init]) {
47    _faceDetectionProcessing = NO;
48    _lastFrameCapturedTimeMilis = 0;
49    _previousFacesCount = -1;
50    _faceDetectorOptions = [EXFaceDetectorUtils mapOptions:options];
51    _timeIntervalMillis = 0;
52    _startDetect = [NSDate new];
53    _interfaceOrientation = UIInterfaceOrientationUnknown;
54  }
55  return self;
56}
57
58# pragma mark Properties setters
59
60- (void)setSession:(AVCaptureSession *)session
61{
62  _session = session;
63}
64
65# pragma mark - JS properties setters
66
67- (void)setIsEnabled:(BOOL)newFaceDetecting
68{
69  // If the data output is already initialized, we toggle its connections instead of adding/removing the output from camera session.
70  // It allows us to smoothly toggle face detection without interrupting preview and reconfiguring camera session.
71  if ([self isDetectingFaceEnabled] != newFaceDetecting) {
72    _faceDetectionEnabled = newFaceDetecting;
73    EX_WEAKIFY(self);
74    [self _runBlockIfQueueIsPresent:^{
75      EX_ENSURE_STRONGIFY(self);
76      if ([self isDetectingFaceEnabled] && ![self isFaceDetecionRunning]) {
77        [self tryEnablingFaceDetection];
78      }
79    }];
80  }
81}
82
83- (void)updateSettings:(NSDictionary *)settings
84{
85  MLKFaceDetectorOptions* newOptions = [EXFaceDetectorUtils newOptions:self.faceDetectorOptions withValues:settings];
86  if(![EXFaceDetectorUtils areOptionsEqual:newOptions to:self.faceDetectorOptions])
87  {
88    self.faceDetectorOptions = newOptions;
89    [self _resetFaceDetector];
90  }
91  if([settings objectForKey:kMinDetectionInterval])
92  {
93    self.timeIntervalMillis = [settings[kMinDetectionInterval] longValue];
94  }
95}
96
97- (void)updateMirrored:(BOOL)mirrored
98{
99  self.mirroredImageSession = mirrored;
100}
101
102# pragma mark - Public API
103
104- (void)maybeStartFaceDetectionOnSession:(AVCaptureSession *)session
105                        withPreviewLayer:(AVCaptureVideoPreviewLayer *)previewLayer
106{
107  [self maybeStartFaceDetectionOnSession:session withPreviewLayer:previewLayer mirrored:NO];
108}
109
110- (void)maybeStartFaceDetectionOnSession:(AVCaptureSession *)session
111                        withPreviewLayer:(AVCaptureVideoPreviewLayer *)previewLayer
112                                mirrored:(BOOL)mirrored
113{
114  _session = session;
115  _mirroredImageSession = mirrored;
116  _previewLayer = previewLayer;
117
118  [self tryEnablingFaceDetection];
119}
120
121- (void)tryEnablingFaceDetection
122{
123  if (!_session) {
124    return;
125  }
126  dispatch_async(dispatch_get_main_queue(), ^{
127    self.interfaceOrientation = [[UIApplication sharedApplication] statusBarOrientation];
128  });
129  [[NSNotificationCenter defaultCenter] addObserver:self
130                                           selector:@selector(handleInterfaceOrientation:)
131                                               name:UIApplicationDidChangeStatusBarOrientationNotification
132                                             object:nil];
133  [_session beginConfiguration];
134
135  if ([self isDetectingFaceEnabled]) {
136    @try {
137      self.faceDetector = [[EXFaceDetector alloc] initWithOptions:_faceDetectorOptions];
138      AVCaptureVideoDataOutput* output = [[AVCaptureVideoDataOutput alloc] init];
139      output.alwaysDiscardsLateVideoFrames = YES;
140      [output setSampleBufferDelegate:self queue:_sessionQueue];
141
142      [self setFaceDetectionRunning:YES];
143      [self _notifyOfFaces:nil withEncoder:nil];
144
145      if([_session canAddOutput:output]) {
146        [_session addOutput:output];
147      } else {
148        EXLogError(@"Unable to add output to camera session! Face detection aborted!");
149      }
150    } @catch (NSException *exception) {
151      EXLogWarn(@"%@", [exception description]);
152    }
153  }
154
155  [_session commitConfiguration];
156}
157
158- (void)handleInterfaceOrientation:(NSNotification *)notificacion
159{
160  self.interfaceOrientation = [[UIApplication sharedApplication] statusBarOrientation];
161}
162
163- (void)stopFaceDetection
164{
165  [self setFaceDetectionRunning:NO];
166  [[NSNotificationCenter defaultCenter] removeObserver:self];
167  if (!_session) {
168    return;
169  }
170
171  [_session beginConfiguration];
172
173  [_session commitConfiguration];
174
175  if ([self isDetectingFaceEnabled]) {
176    _previousFacesCount = -1;
177    [self _notifyOfFaces:nil withEncoder:nil];
178  }
179}
180
181# pragma mark Private API
182
183- (void)_resetFaceDetector
184{
185  [self stopFaceDetection];
186  [self tryEnablingFaceDetection];
187}
188
189- (void)_notifyOfFaces:(NSArray<MLKFace *> *)faces
190           withEncoder:(EXFaceEncoder*)encoder
191{
192  NSArray<MLKFace *> *nonEmptyFaces = faces == nil ? @[] : faces;
193  NSMutableArray<NSDictionary*>* reportableFaces = [NSMutableArray new];
194
195  for(MLKFace* face in nonEmptyFaces)
196  {
197    [reportableFaces addObject:[encoder encode:face]];
198  }
199
200  // Send event when there are faces that have been detected ([faces count] > 0)
201  // or if the listener may think that there are still faces in the video (_prevCount > 0)
202  // or if we really want the event to be sent, eg. to reset listener info (_prevCount == -1).
203  if ([reportableFaces count] > 0 || _previousFacesCount != 0) {
204    if (_onFacesDetected) {
205      _onFacesDetected(reportableFaces);
206    }
207    // Maybe if the delegate is not present anymore we should disable encoding,
208    // however this should never happen.
209
210    _previousFacesCount = [reportableFaces count];
211  }
212}
213
214# pragma mark - Utilities
215
216- (long)_getLongOptionValueForKey:(NSString *)key
217{
218  return [(NSNumber *)[_faceDetectorOptions valueForKey:key] longValue];
219}
220
221- (void)_runBlockIfQueueIsPresent:(void (^)(void))block
222{
223  if (_sessionQueue) {
224    dispatch_async(_sessionQueue, block);
225  }
226}
227
228- (void)captureOutput:(AVCaptureVideoDataOutput *)output
229didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
230       fromConnection:(AVCaptureConnection *)connection
231{
232  NSDate* currentTime = [NSDate new];
233  double timePassedMillis = [currentTime timeIntervalSinceDate:self.startDetect] * 1000;
234  if (timePassedMillis > self.timeIntervalMillis) {
235
236    if (self.faceDetectionProcessing) {
237      return;
238    }
239
240    self.startDetect = currentTime;
241    // This flag is used to drop frames when previous were not processed on time.
242    self.faceDetectionProcessing = YES;
243
244    float outputHeight = [(NSNumber *)output.videoSettings[@"Height"] floatValue];
245    float outputWidth = [(NSNumber *)output.videoSettings[@"Width"] floatValue];
246    if (UIInterfaceOrientationIsPortrait(_interfaceOrientation)) { // We need to inverse width and height in portrait
247      outputHeight = [(NSNumber *)output.videoSettings[@"Width"] floatValue];
248      outputWidth = [(NSNumber *)output.videoSettings[@"Height"] floatValue];
249    }
250    float previewWidth =_previewLayer.bounds.size.width;
251    float previewHeight = _previewLayer.bounds.size.height;
252
253    EXFaceDetectionAngleTransformBlock angleTransform = ^(float angle) { return -angle; };
254
255    CGAffineTransform transformation = [EXCSBufferOrientationCalculator pointTransformForInterfaceOrientation:_interfaceOrientation
256                                                                                               forBufferWidth:outputWidth
257                                                                                              andBufferHeight:outputHeight
258                                                                                                andVideoWidth:previewWidth andVideoHeight:previewHeight
259                                                                                                  andMirrored:_mirroredImageSession];
260
261    UIImageOrientation orientation = [EXFaceDetectorManager imageOrientationFrom:_interfaceOrientation
262                                                                     andMirrored:_mirroredImageSession];
263
264    _startDetect = currentTime;
265    [_faceDetector detectFromBuffer:sampleBuffer
266                        orientation:orientation
267                 completionListener:^(NSArray<MLKFace *> * _Nonnull faces, NSError * _Nonnull error) {
268      if (error != nil) {
269        [self _notifyOfFaces:nil withEncoder:nil];
270      } else {
271        [self _notifyOfFaces:faces
272                 withEncoder:[[EXFaceEncoder alloc] initWithTransform:transformation
273                                                withRotationTransform:angleTransform]];
274      }
275      self.faceDetectionProcessing = NO;
276    }];
277  }
278}
279
280+ (UIImageOrientation)imageOrientationFrom:(UIInterfaceOrientation)orientation
281                               andMirrored:(BOOL)mirrored
282{
283  switch (orientation) {
284    case UIInterfaceOrientationPortrait:
285      return mirrored ? UIImageOrientationLeftMirrored
286                      : UIImageOrientationRight;
287    case UIInterfaceOrientationLandscapeLeft:
288      return mirrored ? UIImageOrientationDownMirrored
289                      : UIImageOrientationUp;
290    case UIInterfaceOrientationPortraitUpsideDown:
291      return mirrored ? UIImageOrientationRightMirrored
292                      : UIImageOrientationLeft;
293    case UIInterfaceOrientationLandscapeRight:
294      return mirrored ? UIImageOrientationUpMirrored
295                      : UIImageOrientationDown;
296    case UIInterfaceOrientationUnknown:
297      return UIImageOrientationUp;
298  }
299}
300
301@end
302