1import * as FaceDetector from 'expo-face-detector';
2import * as ImagePicker from 'expo-image-picker';
3import { ImageInfo } from 'expo-image-picker/build/ImagePicker.types';
4import React from 'react';
5import { Alert, Image, PixelRatio, Platform, ScrollView, StyleSheet, View } from 'react-native';
6
7import { scaledFace, scaledLandmarks } from '../components/Face';
8import ListButton from '../components/ListButton';
9import MonoText from '../components/MonoText';
10
11interface State {
12  selection?: ImagePicker.ImagePickerResult;
13  faceDetection?: {
14    detecting: boolean;
15    faces: FaceDetector.FaceFeature[];
16    image?: FaceDetector.Image;
17    error?: any;
18  };
19}
20
21const imageViewSize = 300;
22
23export default class FaceDetectorScreen extends React.Component<object, State> {
24  static navigationOptions = {
25    title: 'FaceDetector',
26  };
27
28  readonly state: State = {};
29
30  detectFaces = (picture: string) => {
31    this.setState({
32      faceDetection: {
33        detecting: true,
34        faces: [],
35      },
36    });
37    FaceDetector.detectFacesAsync(picture, {
38      mode: FaceDetector.FaceDetectorMode.accurate,
39      detectLandmarks: FaceDetector.FaceDetectorLandmarks.all,
40      runClassifications: FaceDetector.FaceDetectorClassifications.none,
41    })
42      .then((result) => {
43        this.setState({
44          faceDetection: {
45            detecting: false,
46            faces: result.faces,
47            image: result.image,
48          },
49        });
50      })
51      .catch((error) => {
52        this.setState({
53          faceDetection: {
54            detecting: false,
55            faces: [],
56            error,
57          },
58        });
59      });
60  };
61
62  showPicker = async (mediaTypes: ImagePicker.MediaTypeOptions, allowsEditing = false) => {
63    const { granted } = await ImagePicker.requestMediaLibraryPermissionsAsync();
64    if (granted || Platform.OS === 'web') {
65      const result = await ImagePicker.launchImageLibraryAsync({
66        mediaTypes,
67        allowsEditing,
68      });
69      if (result.cancelled) {
70        this.setState({ selection: undefined });
71      } else {
72        this.setState({ selection: result });
73        this.detectFaces(result.uri);
74      }
75    } else {
76      Alert.alert('Permission required!', 'You must allow accessing images in order to proceed.');
77    }
78  };
79
80  render() {
81    return (
82      <ScrollView style={{ padding: 10 }}>
83        <ListButton
84          onPress={() => {
85            this.showPicker(ImagePicker.MediaTypeOptions.Images);
86          }}
87          title="Pick photo"
88        />
89        {this._maybeRenderSelection()}
90        {this._maybeRenderFaceDetection()}
91      </ScrollView>
92    );
93  }
94
95  _maybeRenderSelection = () => {
96    const { selection } = this.state;
97
98    if (!selection || selection.cancelled) {
99      return;
100    }
101
102    return (
103      <View style={styles.sectionContainer}>
104        {!selection || selection.type === 'video' ? null : (
105          <View style={styles.imageContainer}>
106            <Image source={{ uri: selection.uri }} resizeMode="contain" style={styles.image} />
107            {this._maybeRenderDetectedFacesAndLandmarks()}
108          </View>
109        )}
110        <MonoText>{JSON.stringify(selection, null, 2)}</MonoText>
111      </View>
112    );
113  };
114
115  _maybeRenderFaceDetection = () => {
116    const { selection, faceDetection } = this.state;
117
118    if (!selection || selection.cancelled || !faceDetection) {
119      return;
120    }
121
122    if (faceDetection && faceDetection.detecting) {
123      return (
124        <View style={styles.sectionContainer}>
125          <MonoText>Detecting faces…</MonoText>
126        </View>
127      );
128    }
129
130    if (faceDetection && faceDetection.error) {
131      return (
132        <View style={styles.sectionContainer}>
133          <MonoText>Something went wrong: {JSON.stringify(faceDetection.error)}</MonoText>
134        </View>
135      );
136    }
137
138    if (faceDetection && !faceDetection.detecting) {
139      return (
140        <View style={styles.sectionContainer}>
141          <MonoText>Detected faces: {JSON.stringify(faceDetection.faces)}</MonoText>
142          {faceDetection.image && (
143            <MonoText>In image: {JSON.stringify(faceDetection.image)}</MonoText>
144          )}
145        </View>
146      );
147    }
148
149    return null;
150  };
151
152  _maybeRenderDetectedFacesAndLandmarks = () => {
153    const { selection, faceDetection } = this.state;
154    if (selection && !selection?.cancelled && faceDetection) {
155      const { pixelsToDisplayScale } = calculateImageScale(selection);
156      return (
157        <View
158          style={{
159            ...imageOverflowSizeAndPosition(selection),
160            position: 'absolute',
161          }}>
162          {this.state.faceDetection &&
163            this.state.faceDetection.faces.map(scaledFace(pixelsToDisplayScale))}
164          {this.state.faceDetection &&
165            this.state.faceDetection.faces.map(scaledLandmarks(pixelsToDisplayScale))}
166        </View>
167      );
168    }
169    return null;
170  };
171}
172
173const imageOverflowSizeAndPosition = (image: ImageInfo) => {
174  const { scaledImageWidth, scaledImageHeight } = calculateImageScale(image);
175  return {
176    top: (imageViewSize - scaledImageHeight) / 2,
177    left: (imageViewSize - scaledImageWidth) / 2,
178    width: scaledImageWidth,
179    height: scaledImageHeight,
180  };
181};
182
183const calculateImageScale = (image: ImageInfo) => {
184  let scale = 1;
185  const screenMultiplier = PixelRatio.getPixelSizeForLayoutSize(1);
186  const imageHeight = image.height / screenMultiplier;
187  const imageWidth = image.width / screenMultiplier;
188  if (imageWidth > imageHeight) {
189    scale = imageViewSize / imageWidth;
190  } else {
191    scale = imageViewSize / imageHeight;
192  }
193  return {
194    displayScale: scale,
195    pixelsToDisplayScale: scale / screenMultiplier,
196    scaledImageWidth: imageWidth * scale,
197    scaledImageHeight: imageHeight * scale,
198  };
199};
200
201const styles = StyleSheet.create({
202  sectionContainer: {
203    marginVertical: 16,
204    justifyContent: 'center',
205    alignItems: 'center',
206  },
207  imageContainer: {
208    marginBottom: 10,
209    width: imageViewSize,
210    height: imageViewSize,
211  },
212  image: { flex: 1, width: imageViewSize, height: imageViewSize },
213});
214