1import * as FaceDetector from 'expo-face-detector';
2import * as ImagePicker from 'expo-image-picker';
3import { ImageInfo } from 'expo-image-picker/build/ImagePicker.types';
4import React from 'react';
5import { Alert, Image, PixelRatio, Platform, ScrollView, StyleSheet, View } from 'react-native';
6
7import { scaledFace, scaledLandmarks } from '../components/Face';
8import ListButton from '../components/ListButton';
9import MonoText from '../components/MonoText';
10
11interface State {
12  selection?: ImagePicker.ImagePickerResult;
13  faceDetection?: {
14    detecting: boolean;
15    faces: FaceDetector.FaceFeature[];
16    image?: FaceDetector.Image;
17    error?: any;
18  };
19}
20
21const imageViewSize = 300;
22// See: https://github.com/expo/expo/pull/10229#discussion_r490961694
23// eslint-disable-next-line @typescript-eslint/ban-types
24export default class FeceDetectorScreen extends React.Component<{}, State> {
25  static navigationOptions = {
26    title: 'FaceDetector',
27  };
28
29  readonly state: State = {};
30
31  detectFaces = (picture: string) => {
32    this.setState({
33      faceDetection: {
34        detecting: true,
35        faces: [],
36      },
37    });
38    FaceDetector.detectFacesAsync(picture, {
39      mode: FaceDetector.FaceDetectorMode.accurate,
40      detectLandmarks: FaceDetector.FaceDetectorLandmarks.all,
41      runClassifications: FaceDetector.FaceDetectorClassifications.none,
42    })
43      .then((result) => {
44        this.setState({
45          faceDetection: {
46            detecting: false,
47            faces: result.faces,
48            image: result.image,
49          },
50        });
51      })
52      .catch((error) => {
53        this.setState({
54          faceDetection: {
55            detecting: false,
56            faces: [],
57            error,
58          },
59        });
60      });
61  };
62
63  showPicker = async (mediaTypes: ImagePicker.MediaTypeOptions, allowsEditing = false) => {
64    const { granted } = await ImagePicker.requestMediaLibraryPermissionsAsync();
65    if (granted || Platform.OS === 'web') {
66      const result = await ImagePicker.launchImageLibraryAsync({
67        mediaTypes,
68        allowsEditing,
69      });
70      if (result.cancelled) {
71        this.setState({ selection: undefined });
72      } else {
73        this.setState({ selection: result });
74        this.detectFaces(result.uri);
75      }
76    } else {
77      Alert.alert('Permission required!', 'You must allow accessing images in order to proceed.');
78    }
79  };
80
81  render() {
82    return (
83      <ScrollView style={{ padding: 10 }}>
84        <ListButton
85          onPress={() => {
86            this.showPicker(ImagePicker.MediaTypeOptions.Images);
87          }}
88          title="Pick photo"
89        />
90        {this._maybeRenderSelection()}
91        {this._maybeRenderFaceDetection()}
92      </ScrollView>
93    );
94  }
95
96  _maybeRenderSelection = () => {
97    const { selection } = this.state;
98
99    if (!selection || selection.cancelled) {
100      return;
101    }
102
103    return (
104      <View style={styles.sectionContainer}>
105        {!selection || selection.type === 'video' ? null : (
106          <View style={styles.imageContainer}>
107            <Image source={{ uri: selection.uri }} resizeMode="contain" style={styles.image} />
108            {this._maybeRenderDetectedFacesAndLandmarks()}
109          </View>
110        )}
111        <MonoText>{JSON.stringify(selection, null, 2)}</MonoText>
112      </View>
113    );
114  };
115
116  _maybeRenderFaceDetection = () => {
117    const { selection, faceDetection } = this.state;
118
119    if (!selection || selection.cancelled || !faceDetection) {
120      return;
121    }
122
123    if (faceDetection && faceDetection.detecting) {
124      return (
125        <View style={styles.sectionContainer}>
126          <MonoText>Detecting faces…</MonoText>
127        </View>
128      );
129    }
130
131    if (faceDetection && faceDetection.error) {
132      return (
133        <View style={styles.sectionContainer}>
134          <MonoText>Something went wrong: {JSON.stringify(faceDetection.error)}</MonoText>
135        </View>
136      );
137    }
138
139    if (faceDetection && !faceDetection.detecting) {
140      return (
141        <View style={styles.sectionContainer}>
142          <MonoText>Detected faces: {JSON.stringify(faceDetection.faces)}</MonoText>
143          {faceDetection.image && (
144            <MonoText>In image: {JSON.stringify(faceDetection.image)}</MonoText>
145          )}
146        </View>
147      );
148    }
149
150    return null;
151  };
152
153  _maybeRenderDetectedFacesAndLandmarks = () => {
154    const { selection, faceDetection } = this.state;
155    if (selection && faceDetection) {
156      const { pixelsToDisplayScale } = calculateImageScale(selection as ImageInfo);
157      return (
158        <View
159          style={{
160            ...imageOverflowSizeAndPosition(selection as ImageInfo),
161            position: 'absolute',
162          }}>
163          {this.state.faceDetection &&
164            this.state.faceDetection.faces.map(scaledFace(pixelsToDisplayScale))}
165          {this.state.faceDetection &&
166            this.state.faceDetection.faces.map(scaledLandmarks(pixelsToDisplayScale))}
167        </View>
168      );
169    }
170    return null;
171  };
172}
173
174const imageOverflowSizeAndPosition = (image: ImageInfo) => {
175  const { scaledImageWidth, scaledImageHeight } = calculateImageScale(image);
176  return {
177    top: (imageViewSize - scaledImageHeight) / 2,
178    left: (imageViewSize - scaledImageWidth) / 2,
179    width: scaledImageWidth,
180    height: scaledImageHeight,
181  };
182};
183
184const calculateImageScale = (image: ImageInfo) => {
185  let scale = 1;
186  const screenMultiplier = PixelRatio.getPixelSizeForLayoutSize(1);
187  const imageHeight = image.height / screenMultiplier;
188  const imageWidth = image.width / screenMultiplier;
189  if (imageWidth > imageHeight) {
190    scale = imageViewSize / imageWidth;
191  } else {
192    scale = imageViewSize / imageHeight;
193  }
194  return {
195    displayScale: scale,
196    pixelsToDisplayScale: scale / screenMultiplier,
197    scaledImageWidth: imageWidth * scale,
198    scaledImageHeight: imageHeight * scale,
199  };
200};
201
202const styles = StyleSheet.create({
203  sectionContainer: {
204    marginVertical: 16,
205    justifyContent: 'center',
206    alignItems: 'center',
207  },
208  imageContainer: {
209    marginBottom: 10,
210    width: imageViewSize,
211    height: imageViewSize,
212  },
213  image: { flex: 1, width: imageViewSize, height: imageViewSize },
214});
215