diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..57cace115 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM nvcr.io/nvidia/cuda:10.0-cudnn7-runtime-ubuntu18.04 + +RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update \ + && DEBIAN_FRONTEND=noninteractive apt-get -qqy install curl python3-pip python3-tk python3.7-dev ffmpeg git less nano libsm6 libxext6 libxrender-dev \ + && rm -rf /var/lib/apt/lists/* + +# use python3.7 by default +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 1 \ + && update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2 \ + && update-alternatives --set python /usr/bin/python3.7 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 1 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 2 \ + && update-alternatives --set python3 /usr/bin/python3.7 \ + && python -m pip install --upgrade setuptools pip wheel + +ARG PYTORCH_WHEEL="https://download.pytorch.org/whl/cu100/torch-1.0.0-cp37-cp37m-linux_x86_64.whl" +ARG FACE_ALIGNMENT_GIT="git+https://github.com/1adrianb/face-alignment" +ARG AVATARIFY_COMMIT="a300fcaadb6a6964e69d4a90db9e7d72bb87e791" +ARG FOMM_COMMIT="efbe0a6f17b38360ff9a446fddfbb3ce5493534c" + +RUN git clone https://github.com/alievk/avatarify-python.git /app/avatarify && cd /app/avatarify && git checkout ${AVATARIFY_COMMIT} \ + && git clone https://github.com/alievk/first-order-model.git /app/avatarify/fomm && cd /app/avatarify/fomm && git checkout ${FOMM_COMMIT} + +WORKDIR /app/avatarify + +RUN bash scripts/download_data.sh + +RUN pip3 install ${PYTORCH_WHEEL} -r requirements.txt \ + && pip3 install ${PYTORCH_WHEEL} -r fomm/requirements.txt \ + && rm -rf /root/.cache/pip + +ENV PYTHONPATH="/app/avatarify:/app/avatarify/fomm" + +EXPOSE 5557 +EXPOSE 5558 + +CMD ["python3", "afy/cam_fomm.py", "--config", "fomm/config/vox-adv-256.yaml", "--checkpoint", "vox-adv-cpk.pth.tar", "--virt-cam", "9", "--relative", "--adapt_scale", "--is-worker"] diff --git a/README.md b/README.md index fc211e9a4..6aa1f64b4 100644 --- a/README.md +++ b/README.md @@ -1,318 +1,27 @@ ![](docs/mona.gif) -[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) +# Avatarify Python -[](https://join.slack.com/t/avatarify/shared_invite/zt-dyoqy8tc-~4U2ObQ6WoxuwSaWKKVOgg) +Photorealistic avatars for video-conferencing. -:arrow_forward: [Demo](https://youtu.be/Q7LFDT-FRzs) - -:arrow_forward: [AI-generated Elon Musk](https://youtu.be/lONuXGNqLO0) - -# Avatarify - -Photorealistic avatars for video-conferencing [apps](#configure-video-meeting-app). Democratized. +Avatarify Python requires manually downloading and installing some dependencies, and is therefore best suited for users who have some experience with command line applications. [Avatarify Desktop](https://github.com/alievk/avatarify-desktop), which aims to be easier to install and use, is recommended for most users. If you still want to use Avatarify Python, proceed to the [install instructions](docs/). Based on [First Order Motion Model](https://github.com/AliaksandrSiarohin/first-order-model). -Created by: GitHub community. - ## News -- **22 May 2020.** Added [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) mode. Now you can running Avatarify on any computer without GPU! -- **7 May 2020.** Added remote GPU support for all platforms (based on [mynameisfiber's](https://github.com/mynameisfiber) solution). [Demo](https://youtu.be/3Dz_bUIPYFM). Deployment [instructions](https://github.com/alievk/avatarify/wiki/Remote-GPU). +- **7 Mar 2021.** Renamed project to Avatarify Python to distinguish it from other versions of Avatarify +- **14 December 2020.** Released Avatarify Desktop. Check it out [here](https://github.com/alievk/avatarify-desktop). +- **11 July 2020.** Added Docker support. Now you can run Avatarify from Docker on [Linux](https://github.com/alievk/avatarify-python/blob/master/docs/README.md#docker). Thanks to [mikaelhg](https://github.com/mikaelhg) and [mintmaker](https://github.com/mintmaker) for contribution! +- **22 May 2020.** Added [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) mode. Now you can run Avatarify on any computer without GPU! +- **7 May 2020.** Added remote GPU support for all platforms (based on [mynameisfiber's](https://github.com/mynameisfiber) solution). [Demo](https://youtu.be/3Dz_bUIPYFM). Deployment [instructions](https://github.com/alievk/avatarify-python/wiki/Remote-GPU). - **24 April 2020.** Added Windows installation [tutorial](https://www.youtube.com/watch?v=lym9ANVb120). - **17 April 2020.** Created Slack community. Please join via [invitation link](https://join.slack.com/t/avatarify/shared_invite/zt-dyoqy8tc-~4U2ObQ6WoxuwSaWKKVOgg). - **15 April 2020.** Added [StyleGAN-generated](https://www.thispersondoesnotexist.com) avatars. Just press `Q` and now you drive a person that never existed. Every time you push the button – new avatar is sampled. - **13 April 2020.** Added Windows support (kudos to [9of9](https://github.com/9of9)). -## Table of Contents -- [Requirements](#requirements) -- [Install](#install) - - [Download network weights](#download-network-weights) - - [Linux](#linux) - - [Mac](#mac) - - [Windows](#windows) - - [Remote GPU](#remote-gpu) -- [Setup avatars](#setup-avatars) -- [Run](#run) - - [Linux](#linux-1) - - [Mac](#mac-1) - - [Windows](#windows-1) -- [Controls](#controls) -- [Driving your avatar](#driving-your-avatar) -- [Configure video meeting app](#configure-video-meeting-app) - - [Skype](#skype) - - [Zoom](#zoom) - - [Teams](#teams) - - [Slack](#slack) -- [Uninstall](#uninstall) -- [Contribution](#contribution) -- [FAQ](#faq) -- [Troubleshooting](#troubleshooting) - -## Requirements - -You can run Avatarify in two modes: *locally* and *remotely*. - -To run Avatarify *locally* you need a CUDA-enabled (NVIDIA) video card. Otherwise it will fallback to the central processor and run very slowly. These are performance metrics for some hardware: - -- GeForce GTX 1080 Ti: **33 frames per second** -- GeForce GTX 1070: **15 frames per second** -- GeForce GTX 950: **9 frames per second** - -You can also run Avatarify *remotely* on [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) (easy) or on a [dedicated server](https://github.com/alievk/avatarify/wiki/Remote-GPU) with a GPU (harder). There are no special PC requirements for this mode, only a stable internet connection. - -Of course, you also need a webcam! - - - -## Install - -#### Download network weights -Download model's weights from [Dropbox](https://www.dropbox.com/s/t7h24l6wx9vreto/vox-adv-cpk.pth.tar?dl=0), [Yandex.Disk](https://yadi.sk/d/M0FWpz2ExBfgAA) or [Google Drive](https://drive.google.com/file/d/1coUCdyRXDbpWnEkA99NLNY60mb9dQ_n3/view?usp=sharing) [228 MB, md5sum `8a45a24037871c045fbb8a6a8aa95ebc`] - -#### Linux -Linux uses `v4l2loopback` to create virtual camera. - - -1. Download [Miniconda Python 3.7](https://docs.conda.io/en/latest/miniconda.html#linux-installers) and install using command: -```bash -bash Miniconda3-latest-Linux-x86_64.sh -``` -2. Clone `avatarify` and install its dependencies (sudo privelege is required): -```bash -git clone https://github.com/alievk/avatarify.git -cd avatarify -bash scripts/install.sh -``` -3. [Download network weights](#download-network-weights) and place `vox-adv-cpk.pth.tar` file in the `avatarify` directory (don't unpack it). - -#### Mac -*(!) Note*: we found out that in versions after [v4.6.8 (March 23, 2020)](https://zoom.us/client/4.6.19178.0323/ZoomInstaller.pkg) Zoom disabled support for virtual cameras on Mac. To use Avatarify in Zoom you can choose from 2 options: -- Install [Zoom v4.6.8](https://zoom.us/client/4.6.19178.0323/ZoomInstaller.pkg) which is the last version that supports virtual cameras -- Use latest version of Zoom, but disable library validation by running this command in terminal: -```bash -codesign --remove-signature /Applications/zoom.us.app -``` - - - -We will use [CamTwist](http://camtwiststudio.com) to create virtual camera for Mac. - -1. Install [Miniconda Python 3.7](https://docs.conda.io/en/latest/miniconda.html#macosx-installers) or use *Homebrew Cask*: `brew cask install miniconda`. -2. [Download](https://github.com/alievk/avatarify/archive/master.zip) and unpack the repository or use `git`: -```bash -git clone https://github.com/alievk/avatarify.git -cd avatarify -bash scripts/install_mac.sh -``` -3. Download and install [CamTwist](http://camtwiststudio.com) from [here](http://camtwiststudio.com/download). It's easy. - -#### Windows - -:arrow_forward: [Video tutorial](https://youtu.be/lym9ANVb120) - -This guide is tested for Windows 10. - - -1. Install [Miniconda Python 3.7](https://docs.conda.io/en/latest/miniconda.html#windows-installers). -2. Install [Git](https://git-scm.com/download/win). -3. Press Windows button and type "miniconda". Run suggested Anaconda Prompt. -4. Download and install Avatarify (please copy-paste these commands and don't change them): -```bash -git clone https://github.com/alievk/avatarify.git -cd avatarify -scripts\install_windows.bat -``` -5. [Download network weights](#download-network-weights) and place `vox-adv-cpk.pth.tar` file in the `avatarify` directory (don't unpack it). -6. Run `run_windows.bat`. If installation was successful, two windows "cam" and "avatarify" will appear. Leave these windows open for the next installation steps. -7. Install [OBS Studio](https://obsproject.com/) for capturing Avatarify output. -8. Install [VirtualCam plugin](https://obsproject.com/forum/resources/obs-virtualcam.539/). Choose `Install and register only 1 virtual camera`. -9. Run OBS Studio. -10. In the Sources section, press on Add button ("+" sign), select Windows Capture and press OK. In the appeared window, choose "[python.exe]: avatarify" in Window drop-down menu and press OK. Then select Edit -> Transform -> Fit to screen. -11. In OBS Studio, go to Tools -> VirtualCam. Check AutoStart, set Buffered Frames to 0 and press Start. -12. Now `OBS-Camera` camera should be available in Zoom (or other videoconferencing software). - -The steps 10-11 are required only once during setup. - -#### Remote GPU - -You can offload the heavy work to [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) or a [server with a GPU](https://github.com/alievk/avatarify/wiki/Remote-GPU) and use your laptop just to communicate the video stream. - -## Setup avatars -Avatarify comes with a standard set of avatars of famous people, but you can extend this set simply copying your avatars into `avatars` folder. - -Follow these advices for better visual quality: -* Make square crop of your avatar picture. -* Crop avatar's face so that it's not too close not too far. Use standard avarars as reference. -* Prefer pictures with uniform background. It will diminish visual artifacts. - -## Run -Your web cam must be plugged-in. - -**Note:** run your video-conferencing app only after Avatarify is started. - -#### Linux -The run script will create virtual camera `/dev/video9`. You can change these settings in `scripts/settings.sh`. - - -You can use command `v4l2-ctl --list-devices` to list all devices in your system. - -Run: -```bash -bash run.sh -``` - -`cam` and `avatarify` windows will pop-up. The `cam` window is for controlling your face position and `avatarify` is for the avatar animation preview. Please follow these [recommendations](#driving-your-avatar) to drive your avatars. - -#### Mac -*Note*: On Mac Avatarify runs only with [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) or a [dedicated server](https://github.com/alievk/avatarify/wiki/Remote-GPU) with GPU. - -Please find where you downloaded `avatarify` and substitute path `/path/to/avatarify` below. - - -1. To run Avatarify please follow instructions for [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) or a [dedicated server](https://github.com/alievk/avatarify/wiki/Remote-GPU). -2. Go to [CamTwist](http://camtwiststudio.com). -3. Choose `Desktop+` and press `Select`. -4. In the `Settings` section choose `Confine to Application Window` and select `python (avatarify)` from the drop-down menu. - -`cam` and `avatarify` windows will pop-up. The `cam` window is for controlling your face position and `avatarify` is for the avatar animation preview. Please follow these [recommendations](#driving-your-avatar) to drive your avatars. - -#### Windows - - - -1. In Anaconda Prompt: -``` -cd C:\path\to\avatarify -run_windows.bat -``` -2. Run OBS Studio. It should automaitcally start streaming video from Avatarify to `OBS-Camera`. - -`cam` and `avatarify` windows will pop-up. The `cam` window is for controlling your face position and `avatarify` is for the avatar animation preview. Please follow these [recommendations](#driving-your-avatar) to drive your avatars. - -**Note:** To reduce video latency, in OBS Studio right click on the preview window and uncheck Enable Preview. - -## Controls - - -Keys | Controls ---- | --- -1-9 | These will immediately switch between the first 9 avatars. -Q | Turns on StyleGAN-generated avatar. Every time you push the button – new avatar is sampled. -0 | Toggles avatar display on and off. -A/D | Previous/next avatar in folder. -W/S | Zoom camera in/out. -U/H/J/K | Translate camera. `H` - left, `K` - right, `U` - up, `J` - Down by 5 pixels. Add `Shift` to adjust by 1 pixel. -Shift-Z | Reset camera zoom and translation -Z/C | Adjust avatar target overlay opacity. -X | Reset reference frame. -F | Toggle reference frame search mode. -R | Mirror reference window. -T | Mirror output window. -L | Reload avatars. -I | Show FPS -ESC | Quit - -## Driving your avatar - -These are the main principles for driving your avatar: - -* Align your face in the camera window as closely as possible in proportion and position to the target avatar. Use zoom in/out function (W/S keys) and camera left, right, up, down translation (U/H/J/K keys). When you have aligned, hit 'X' to use this frame as reference to drive the rest of the animation -* Use the overlay function (Z/C keys) to match your and avatar's face expressions as close as possible - -Alternatively, you can hit 'F' for the software to attempt to find a better reference frame itself. This will slow down the framerate, but while this is happening, you can keep moving your head around: the preview window will flash green when it finds your facial pose is a closer match to the avatar than the one it is currently using. You will see two numbers displayed as well: the first number is how closely you are currently aligned to the avatar, and the second number is how closely the reference frame is aligned. - -You want to get the first number as small as possible - around 10 is usually a good alignment. When you are done, press 'F' again to exit reference frame search mode. - -You don't need to be exact, and some other configurations can yield better results still, but it's usually a good starting point. - -## Configure video meeting app - -Avatarify supports any video-conferencing app where video input source can be changed (Zoom, Skype, Hangouts, Slack, ...). Here are a few examples how to configure particular app to use Avatarify. - -### Skype - -Go to Settings -> Audio & Video, choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) camera. - - - -### Zoom - -Go to Settings -> Video and choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) from Camera drop-down menu. - - - -### Teams - -Go to your profile picture -> Settings -> Devices and choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) from Camera drop-down menu. - - - -### Slack - -Make a call, allow browser using cameras, click on Settings icon, choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) in Video settings drop-down menu. - - - - -## Uninstall -To remove Avatarify and its related programs follow the [instructions](https://github.com/alievk/avatarify/wiki/Removing-Avatarify) in the Wiki. - - -## Contribution - -Our goal is to democratize photorealistic avatars for video-conferencing. To make the technology even more accessible, we have to tackle the following problems: - -* ~~Add support for more platforms (Linux and Mac are already supported).~~ -* ~~Remote GPU support. This is a work in progress.~~ -* Porting to non-CUDA GPUs (Intel integrated GPUs, AMD GPUs, etc) and optimization. The goal is to run Avatarify real-time (at least 10FPS) on modern laptops. - -Please make pull requests if you have any improvements or bug-fixes. - - -## FAQ - -Q: **Do I need any knowledge of programming to run Avatarify?** -A: Not really, but you need some beginner-level knowledge of the command line. For Windows we recorded a video [tutorial](https://www.youtube.com/watch?v=lym9ANVb120), so it’ll be easy to install. - -Q: **Why does it work so slow on my Macbook?** -A: The model used in Avatarify requires a CUDA-enabled NVIDIA GPU to perform heavy computations. Macbooks don’t have such GPUs, and for processing use CPU, which has much less computing power to run Avatarify smoothly. - -Q: **I don’t have a NVIDIA GPU, can I run it?** -A: You still can run it without a NVIDIA GPU, but with drastically reduced performance (<1fps). - -Q: **I have an ATI GPU (e.g. Radeon). Why does it work so slow?** -A: To run the neural network Avatarify uses PyTorch library, which is optimized for CUDA. If PyTorch can’t find a CUDA-enabled GPU in your system it will fallback to CPU. The performance on the CPU will be much worse. - -Q: **How to add a new avatar?** -A: It’s easy. All you need is to find a picture of your avatar and put it in the `avatars` folder. [More](https://github.com/alievk/avatarify#setup-avatars). - -Q: **My avatar looks distorted.** -A: You need to calibrate your face position. Please follow the [tips](https://github.com/alievk/avatarify#driving-your-avatar) or watch the video [tutorial](https://youtu.be/lym9ANVb120?t=662). - -Q: **Can I use a cloud GPU?** -A: This is work in progress. See the relevant [discussion](https://github.com/alievk/avatarify/issues/115). - -Q: **Avatarify crashed, what to do?** -A: First, try to find your error in the [troubleshooting](https://github.com/alievk/avatarify#troubleshooting) section. If it is not there, try to find it in the [issues](https://github.com/alievk/avatarify/issues). If you couldn’t find your issue there, please open a new one using the issue template. - -Q: **Can I use Avatarify for commercial purposes?** -A: No. Avatarify and First Order Motion Model are licensed under Creative Commons Non-Commercial license, which prohibits commercial use. - -Q: **What video conferencing apps does Avatarify support?** -A: Avatarify creates a virtual camera which can be plugged into any app where video input source can be changed (Zoom, Skype, Hangouts, Slack, ...). - -Q: **Where can I discuss Avatarify-related topics with the community?** -A: We have Slack. Please join: [](https://join.slack.com/t/avatarify/shared_invite/zt-dyoqy8tc-~4U2ObQ6WoxuwSaWKKVOgg) - +## Avatarify apps -## Troubleshooting +We have deployed Avatarify on iOS and Android devices using our proprietary inference engine. The iOS version features the Life mode for recording animations in real time. However, the Life mode is not available on Android devices due to the diversity of the devices we have to support. -Please follow the [Wiki](https://github.com/alievk/avatarify/wiki/Troubleshooting) page. +[drawing](https://apps.apple.com/app/apple-store/id1512669147?pt=121960189&ct=GitHub&mt=8) +[drawing](https://play.google.com/store/apps/details?id=com.avatarify.android) diff --git a/afy/cam_fomm.py b/afy/cam_fomm.py index 35c1535a5..13ac1d4a3 100644 --- a/afy/cam_fomm.py +++ b/afy/cam_fomm.py @@ -16,6 +16,8 @@ log = Tee('./var/log/cam_fomm.log') +# Where to split an array from face_alignment to separate each landmark +LANDMARK_SLICE_ARRAY = np.array([17, 22, 27, 31, 36, 42, 48, 60]) if _platform == 'darwin': if not opt.is_client: @@ -24,7 +26,7 @@ exit() -def is_new_frame_better(source, driving, precitor): +def is_new_frame_better(source, driving, predictor): global avatar_kp global display_string @@ -72,6 +74,10 @@ def load_images(IMG_SIZE = 256): for i, f in enumerate(images_list): if f.endswith('.jpg') or f.endswith('.jpeg') or f.endswith('.png'): img = cv2.imread(f) + if img is None: + log("Failed to open image: {}".format(f)) + continue + if img.ndim == 2: img = np.tile(img[..., None], [1, 1, 3]) img = img[..., :3][..., ::-1] @@ -96,6 +102,13 @@ def draw_rect(img, rw=0.6, rh=0.8, color=(255, 0, 0), thickness=2): d = h - u img = cv2.rectangle(img, (int(l), int(u)), (int(r), int(d)), color, thickness) +def kp_to_pixels(arr): + '''Convert normalized landmark locations to screen pixels''' + return ((arr + 1) * 127).astype(np.int32) + +def draw_face_landmarks(img, face_kp, color=(20, 80, 255)): + if face_kp is not None: + img = cv2.polylines(img, np.split(kp_to_pixels(face_kp), LANDMARK_SLICE_ARRAY), False, color) def print_help(): info('\n\n=== Control keys ===') @@ -116,6 +129,31 @@ def print_help(): info('\n\n') +def draw_fps(frame, fps, timing, x0=10, y0=20, ystep=30, fontsz=0.5, color=(255, 255, 255)): + frame = frame.copy() + cv2.putText(frame, f"FPS: {fps:.1f}", (x0, y0 + ystep * 0), 0, fontsz * IMG_SIZE / 256, color, 1) + cv2.putText(frame, f"Model time (ms): {timing['predict']:.1f}", (x0, y0 + ystep * 1), 0, fontsz * IMG_SIZE / 256, color, 1) + cv2.putText(frame, f"Preproc time (ms): {timing['preproc']:.1f}", (x0, y0 + ystep * 2), 0, fontsz * IMG_SIZE / 256, color, 1) + cv2.putText(frame, f"Postproc time (ms): {timing['postproc']:.1f}", (x0, y0 + ystep * 3), 0, fontsz * IMG_SIZE / 256, color, 1) + return frame + + +def draw_landmark_text(frame, thk=2, fontsz=0.5, color=(0, 0, 255)): + frame = frame.copy() + cv2.putText(frame, "ALIGN FACES", (60, 20), 0, fontsz * IMG_SIZE / 255, color, thk) + cv2.putText(frame, "THEN PRESS X", (60, 245), 0, fontsz * IMG_SIZE / 255, color, thk) + return frame + + +def draw_calib_text(frame, thk=2, fontsz=0.5, color=(0, 0, 255)): + frame = frame.copy() + cv2.putText(frame, "FIT FACE IN RECTANGLE", (40, 20), 0, fontsz * IMG_SIZE / 255, color, thk) + cv2.putText(frame, "W - ZOOM IN", (60, 40), 0, fontsz * IMG_SIZE / 255, color, thk) + cv2.putText(frame, "S - ZOOM OUT", (60, 60), 0, fontsz * IMG_SIZE / 255, color, thk) + cv2.putText(frame, "THEN PRESS X", (60, 245), 0, fontsz * IMG_SIZE / 255, color, thk) + return frame + + def select_camera(config): cam_config = config['cam_config'] cam_id = None @@ -128,7 +166,10 @@ def select_camera(config): cam_frames = cam_selector.query_cameras(config['query_n_cams']) if cam_frames: - cam_id = cam_selector.select_camera(cam_frames, window="CLICK ON YOUR CAMERA") + if len(cam_frames) == 1: + cam_id = list(cam_frames)[0] + else: + cam_id = cam_selector.select_camera(cam_frames, window="CLICK ON YOUR CAMERA") log(f"Selected camera {cam_id}") with open(cam_config, 'w') as f: @@ -189,6 +230,9 @@ def select_camera(config): enable_vcam = not opt.no_stream + ret, frame = cap.read() + stream_img_size = frame.shape[1], frame.shape[0] + if enable_vcam: if _platform in ['linux', 'linux2']: try: @@ -197,15 +241,13 @@ def select_camera(config): log("pyfakewebcam is not installed.") exit(1) - ret, frame = cap.read() - stream_img_size = frame.shape[1], frame.shape[0] stream = pyfakewebcam.FakeWebcam(f'/dev/video{opt.virt_cam}', *stream_img_size) else: enable_vcam = False - log("Virtual camera is supported only on Linux.") + # log("Virtual camera is supported only on Linux.") - if not enable_vcam: - log("Virtual camera streaming will be disabled.") + # if not enable_vcam: + # log("Virtual camera streaming will be disabled.") cur_ava = 0 avatar = None @@ -225,6 +267,8 @@ def select_camera(config): find_keyframe = False is_calibrated = False + show_landmarks = False + fps_hist = [] fps = 0 show_fps = False @@ -253,8 +297,8 @@ def select_camera(config): frame = frame[..., ::-1] frame_orig = frame.copy() - frame, lrudwh = crop(frame, p=frame_proportion, offset_x=frame_offset_x, offset_y=frame_offset_y) - frame_lrudwh = lrudwh + frame, (frame_offset_x, frame_offset_y) = crop(frame, p=frame_proportion, offset_x=frame_offset_x, offset_y=frame_offset_y) + frame = resize(frame, (IMG_SIZE, IMG_SIZE))[..., :3] if find_keyframe: @@ -280,6 +324,11 @@ def select_camera(config): key = cv2.waitKey(1) + if cv2.getWindowProperty('cam', cv2.WND_PROP_VISIBLE) < 1.0: + break + elif is_calibrated and cv2.getWindowProperty('avatarify', cv2.WND_PROP_VISIBLE) < 1.0: + break + if key == 27: # ESC break elif key == ord('d'): @@ -301,29 +350,21 @@ def select_camera(config): frame_proportion += 0.05 frame_proportion = min(frame_proportion, 1.0) elif key == ord('H'): - if frame_lrudwh[0] - 1 > 0: - frame_offset_x -= 1 + frame_offset_x -= 1 elif key == ord('h'): - if frame_lrudwh[0] - 5 > 0: - frame_offset_x -= 5 + frame_offset_x -= 5 elif key == ord('K'): - if frame_lrudwh[1] + 1 < frame_lrudwh[4]: - frame_offset_x += 1 + frame_offset_x += 1 elif key == ord('k'): - if frame_lrudwh[1] + 5 < frame_lrudwh[4]: - frame_offset_x += 5 + frame_offset_x += 5 elif key == ord('J'): - if frame_lrudwh[2] - 1 > 0: - frame_offset_y -= 1 + frame_offset_y -= 1 elif key == ord('j'): - if frame_lrudwh[2] - 5 > 0: - frame_offset_y -= 5 + frame_offset_y -= 5 elif key == ord('U'): - if frame_lrudwh[3] + 1 < frame_lrudwh[5]: - frame_offset_y += 1 + frame_offset_y += 1 elif key == ord('u'): - if frame_lrudwh[3] + 5 < frame_lrudwh[5]: - frame_offset_y += 5 + frame_offset_y += 5 elif key == ord('Z'): frame_offset_x = 0 frame_offset_y = 0 @@ -336,6 +377,7 @@ def select_camera(config): cv2.moveWindow('avatarify', 600, 250) is_calibrated = True + show_landmarks = False elif key == ord('z'): overlay_alpha = max(overlay_alpha - 0.1, 0.0) elif key == ord('c'): @@ -346,6 +388,8 @@ def select_camera(config): output_flip = not output_flip elif key == ord('f'): find_keyframe = not find_keyframe + elif key == ord('o'): + show_landmarks = not show_landmarks elif key == ord('q'): try: log('Loading StyleGAN avatar...') @@ -357,7 +401,7 @@ def select_camera(config): elif key == ord('l'): try: log('Reloading avatars...') - avatars = load_images() + avatars, avatar_names = load_images() passthrough = False log("Images reloaded") except: @@ -374,9 +418,17 @@ def select_camera(config): log(key) if overlay_alpha > 0: - preview_frame = cv2.addWeighted( avatars[cur_ava], overlay_alpha, frame, 1.0 - overlay_alpha, 0.0) + preview_frame = cv2.addWeighted( avatar, overlay_alpha, frame, 1.0 - overlay_alpha, 0.0) else: preview_frame = frame.copy() + + if show_landmarks: + # Dim the background to make it easier to see the landmarks + preview_frame = cv2.convertScaleAbs(preview_frame, alpha=0.5, beta=0.0) + + draw_face_landmarks(preview_frame, avatar_kp, (200, 20, 10)) + frame_kp = predictor.get_frame_kp(frame) + draw_face_landmarks(preview_frame, frame_kp) if preview_flip: preview_frame = cv2.flip(preview_frame, 1) @@ -393,17 +445,12 @@ def select_camera(config): preview_frame = cv2.putText(preview_frame, display_string, (10, 220), 0, 0.5 * IMG_SIZE / 256, (255, 255, 255), 1) if show_fps: - timing_string = f"FPS/Model/Pre/Post: {fps:.1f} / {timing['predict']:.1f} / {timing['preproc']:.1f} / {timing['postproc']:.1f}" - preview_frame = cv2.putText(preview_frame, timing_string, (10, 240), 0, 0.3 * IMG_SIZE / 256, (255, 255, 255), 1) + preview_frame = draw_fps(preview_frame, fps, timing) if not is_calibrated: - color = (0, 0, 255) - thk = 2 - fontsz = 0.5 - preview_frame = cv2.putText(preview_frame, "FIT FACE IN RECTANGLE", (40, 20), 0, fontsz * IMG_SIZE / 255, color, thk) - preview_frame = cv2.putText(preview_frame, "W - ZOOM IN", (60, 40), 0, fontsz * IMG_SIZE / 255, color, thk) - preview_frame = cv2.putText(preview_frame, "S - ZOOM OUT", (60, 60), 0, fontsz * IMG_SIZE / 255, color, thk) - preview_frame = cv2.putText(preview_frame, "THEN PRESS X", (60, 245), 0, fontsz * IMG_SIZE / 255, color, thk) + preview_frame = draw_calib_text(preview_frame) + elif show_landmarks: + preview_frame = draw_landmark_text(preview_frame) if not opt.hide_rect: draw_rect(preview_frame) diff --git a/afy/cam_fomm.py.orig b/afy/cam_fomm.py.orig deleted file mode 100644 index 5fd631d2a..000000000 --- a/afy/cam_fomm.py.orig +++ /dev/null @@ -1,374 +0,0 @@ -import os, sys -import glob -import yaml -import time -import requests - -import numpy as np -import cv2 - -from afy.videocaptureasync import VideoCaptureAsync -from afy.arguments import opt -from afy.utils import Once, log, crop, pad_img, resize, TicToc - - -from sys import platform as _platform -_streaming = False -if _platform == 'linux' or _platform == 'linux2': - import pyfakewebcam - _streaming = True - - -if _platform == 'darwin': - if opt.worker_host is None: - log('\nOnly remote GPU mode is supported for Mac (use --worker-host option to connect to the server)') - log('Standalone version will be available lately!\n') - exit() - - -def is_new_frame_better(source, driving, precitor): - global avatar_kp - global display_string - - if avatar_kp is None: - display_string = "No face detected in avatar." - return False - - if predictor.get_start_frame() is None: - display_string = "No frame to compare to." - return True - - driving_smaller = resize(driving, (128, 128))[..., :3] - new_kp = predictor.get_frame_kp(driving) - - if new_kp is not None: - new_norm = (np.abs(avatar_kp - new_kp) ** 2).sum() - old_norm = (np.abs(avatar_kp - predictor.get_start_frame_kp()) ** 2).sum() - - out_string = "{0} : {1}".format(int(new_norm * 100), int(old_norm * 100)) - display_string = out_string - log(out_string) - - return new_norm < old_norm - else: - display_string = "No face found!" - return False - - -def load_stylegan_avatar(): - url = "https://thispersondoesnotexist.com/image" - r = requests.get(url, headers={'User-Agent': "My User Agent 1.0"}).content - - image = np.frombuffer(r, np.uint8) - image = cv2.imdecode(image, cv2.IMREAD_COLOR) - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - - image = resize(image, (IMG_SIZE, IMG_SIZE)) - - return image - - -def change_avatar(predictor, new_avatar): - global avatar, avatar_kp, kp_source - avatar_kp = predictor.get_frame_kp(new_avatar) - kp_source = None - avatar = new_avatar - predictor.set_source_image(avatar) - - -def draw_rect(img, rw=0.6, rh=0.8, color=(255, 0, 0), thickness=2): - h, w = img.shape[:2] - l = w * (1 - rw) // 2 - r = w - l - u = h * (1 - rh) // 2 - d = h - u - img = cv2.rectangle(img, (int(l), int(u)), (int(r), int(d)), color, thickness) - - -def print_help(): - log('\n\n=== Control keys ===') - log('1-9: Change avatar') - log('W: Zoom camera in') - log('S: Zoom camera out') - log('A: Previous avatar in folder') - log('D: Next avatar in folder') - log('Q: Get random avatar') - log('X: Calibrate face pose') - log('I: Show FPS') - log('ESC: Quit') - log('\nFull key list: https://github.com/alievk/avatarify#controls') - log('\n\n') - -if __name__ == "__main__": - - global display_string - display_string = "" - - IMG_SIZE = 256 - - if opt.no_stream: - log('Force no streaming') - _streaming = False - - log('Loading Predictor') - predictor_args = { - 'config_path': opt.config, - 'checkpoint_path': opt.checkpoint, - 'relative': opt.relative, - 'adapt_movement_scale': opt.adapt_scale, - 'enc_downscale': opt.enc_downscale - } - if opt.is_worker: - from afy import predictor_worker - predictor_worker.run_worker(opt.worker_port) - sys.exit(0) - elif opt.worker_host: - from afy import predictor_remote - predictor = predictor_remote.PredictorRemote( - worker_host=opt.worker_host, worker_port=opt.worker_port, - **predictor_args - ) - else: - from afy import predictor_local - predictor = predictor_local.PredictorLocal( - **predictor_args - ) - - avatars=[] - images_list = sorted(glob.glob(f'{opt.avatars}/*')) - for i, f in enumerate(images_list): - if f.endswith('.jpg') or f.endswith('.jpeg') or f.endswith('.png'): - key = len(avatars) + 1 - log(f'Key {key}: {f}') - img = cv2.imread(f) - if img.ndim == 2: - img = np.tile(img[..., None], [1, 1, 3]) - img = img[..., :3][..., ::-1] - img = resize(img, (IMG_SIZE, IMG_SIZE)) - avatars.append(img) - - - cap = VideoCaptureAsync(opt.cam) - cap.start() - - if _streaming: - ret, frame = cap.read() - stream_img_size = frame.shape[1], frame.shape[0] - stream = pyfakewebcam.FakeWebcam(f'/dev/video{opt.virt_cam}', *stream_img_size) - - cur_ava = 0 - avatar = None - change_avatar(predictor, avatars[cur_ava]) - passthrough = False - - cv2.namedWindow('cam', cv2.WINDOW_GUI_NORMAL) - cv2.moveWindow('cam', 500, 250) - - frame_proportion = 0.9 - frame_offset_x = 0 - frame_offset_y = 0 - - overlay_alpha = 0.0 - preview_flip = False - output_flip = False - find_keyframe = False - is_calibrated = False - - fps_hist = [] - fps = 0 - show_fps = False - - print_help() - - try: - while True: - tt = TicToc() - - timing = { - 'preproc': 0, - 'predict': 0, - 'postproc': 0 - } - - green_overlay = False - - tt.tic() - - ret, frame = cap.read() - if not ret: - log("Can't receive frame (stream end?). Exiting ...") - break - - frame = frame[..., ::-1] - frame_orig = frame.copy() - - frame, lrudwh = crop(frame, p=frame_proportion, offset_x=frame_offset_x, offset_y=frame_offset_y) - frame_lrudwh = lrudwh - frame = resize(frame, (IMG_SIZE, IMG_SIZE))[..., :3] - - if find_keyframe: - if is_new_frame_better(avatar, frame, predictor): - log("Taking new frame!") - green_overlay = True - predictor.reset_frames() - - timing['preproc'] = tt.toc() - - if passthrough: - out = frame - else: - tt.tic() - pred = predictor.predict(frame) - out = pred - timing['predict'] = tt.toc() - - tt.tic() - - if not opt.no_pad: - out = pad_img(out, stream_img_size) - - key = cv2.waitKey(1) - - if key == 27: # ESC - break - elif key == ord('d'): - cur_ava += 1 - if cur_ava >= len(avatars): - cur_ava = 0 - passthrough = False - change_avatar(predictor, avatars[cur_ava]) - elif key == ord('a'): - cur_ava -= 1 - if cur_ava < 0: - cur_ava = len(avatars) - 1 - passthrough = False - change_avatar(predictor, avatars[cur_ava]) - elif key == ord('w'): - frame_proportion -= 0.05 - frame_proportion = max(frame_proportion, 0.1) - elif key == ord('s'): - frame_proportion += 0.05 - frame_proportion = min(frame_proportion, 1.0) - elif key == ord('H'): - if frame_lrudwh[0] - 1 > 0: - frame_offset_x -= 1 - elif key == ord('h'): - if frame_lrudwh[0] - 5 > 0: - frame_offset_x -= 5 - elif key == ord('K'): - if frame_lrudwh[1] + 1 < frame_lrudwh[4]: - frame_offset_x += 1 - elif key == ord('k'): - if frame_lrudwh[1] + 5 < frame_lrudwh[4]: - frame_offset_x += 5 - elif key == ord('J'): - if frame_lrudwh[2] - 1 > 0: - frame_offset_y -= 1 - elif key == ord('j'): - if frame_lrudwh[2] - 5 > 0: - frame_offset_y -= 5 - elif key == ord('U'): - if frame_lrudwh[3] + 1 < frame_lrudwh[5]: - frame_offset_y += 1 - elif key == ord('u'): - if frame_lrudwh[3] + 5 < frame_lrudwh[5]: - frame_offset_y += 5 - elif key == ord('Z'): - frame_offset_x = 0 - frame_offset_y = 0 - frame_proportion = 0.9 - elif key == ord('x'): - predictor.reset_frames() - - if not is_calibrated: - cv2.namedWindow('avatarify', cv2.WINDOW_GUI_NORMAL) - cv2.moveWindow('avatarify', 600, 250) - - is_calibrated = True - elif key == ord('z'): - overlay_alpha = max(overlay_alpha - 0.1, 0.0) - elif key == ord('c'): - overlay_alpha = min(overlay_alpha + 0.1, 1.0) - elif key == ord('r'): - preview_flip = not preview_flip - elif key == ord('t'): - output_flip = not output_flip - elif key == ord('f'): - find_keyframe = not find_keyframe - elif key == ord('q'): - try: - log('Loading StyleGAN avatar...') - avatar = load_stylegan_avatar() - passthrough = False - change_avatar(predictor, avatar) - except: - log('Failed to load StyleGAN avatar') - elif key == ord('i'): - show_fps = not show_fps - elif 48 < key < 58: - cur_ava = min(key - 49, len(avatars) - 1) - passthrough = False - change_avatar(predictor, avatars[cur_ava]) - elif key == 48: - passthrough = not passthrough - elif key != -1: - log(key) - - if _streaming: - out = resize(out, stream_img_size) - stream.schedule_frame(out) - - if overlay_alpha > 0: - preview_frame = cv2.addWeighted( avatars[cur_ava], overlay_alpha, frame, 1.0 - overlay_alpha, 0.0) - else: - preview_frame = frame.copy() - - if preview_flip: - preview_frame = cv2.flip(preview_frame, 1) - - if output_flip: - out = cv2.flip(out, 1) - - if green_overlay: - green_alpha = 0.8 - overlay = preview_frame.copy() - overlay[:] = (0, 255, 0) - preview_frame = cv2.addWeighted( preview_frame, green_alpha, overlay, 1.0 - green_alpha, 0.0) - - timing['postproc'] = tt.toc() - - if find_keyframe: - preview_frame = cv2.putText(preview_frame, display_string, (10, 220), 0, 0.5 * IMG_SIZE / 256, (255, 255, 255), 1) - - if show_fps: - timing_string = f"FPS/Model/Pre/Post: {fps:.1f} / {timing['predict']:.1f} / {timing['preproc']:.1f} / {timing['postproc']:.1f}" - preview_frame = cv2.putText(preview_frame, timing_string, (10, 240), 0, 0.3 * IMG_SIZE / 256, (255, 255, 255), 1) - - if not is_calibrated: - color = (0, 0, 255) - thk = 2 - fontsz = 0.5 - preview_frame = cv2.putText(preview_frame, "FIT FACE IN RECTANGLE", (40, 20), 0, fontsz * IMG_SIZE / 255, color, thk) - preview_frame = cv2.putText(preview_frame, "W - ZOOM IN", (60, 40), 0, fontsz * IMG_SIZE / 255, color, thk) - preview_frame = cv2.putText(preview_frame, "S - ZOOM OUT", (60, 60), 0, fontsz * IMG_SIZE / 255, color, thk) - preview_frame = cv2.putText(preview_frame, "THEN PRESS X", (60, 245), 0, fontsz * IMG_SIZE / 255, color, thk) - - if not opt.hide_rect: - draw_rect(preview_frame) - - # cv2.imshow('cam', preview_frame[..., ::-1]) - if is_calibrated: - cv2.destroyWindow('cam') - cv2.imshow('avatarify', out[..., ::-1]) - else: - cv2.imshow('cam', preview_frame[..., ::-1]) - - fps_hist.append(tt.toc(total=True)) - if len(fps_hist) == 10: - fps = 10 / (sum(fps_hist) / 1000) - fps_hist = [] - except KeyboardInterrupt: - pass - - cap.stop() - cv2.destroyAllWindows() diff --git a/afy/camera_selector.py b/afy/camera_selector.py index 9b5100853..897562d01 100644 --- a/afy/camera_selector.py +++ b/afy/camera_selector.py @@ -46,7 +46,7 @@ def make_grid(images, cell_size=(320, 240), cols=2): # add rect img = cv2.rectangle(img, (1, 1), (w0 - 1, h0 - 1), (0, 0, 255), 2) # add id - img = cv2.putText(img, f'{camid}', (10, 30), 0, 1, (0, 255, 0), 2) + img = cv2.putText(img, f'Camera {camid}', (10, 30), 0, 1, (0, 255, 0), 2) c = i % cols r = i // cols grid[r * h0:(r + 1) * h0, c * w0:(c + 1) * w0] = img[..., :3] @@ -64,11 +64,18 @@ def mouse_callback(event, x, y, flags, userdata): g_selected_cam = camid -def select_camera(cam_frames, window="cameras"): +def select_camera(cam_frames, window="Camera selector"): cell_size = 320, 240 grid_cols = 2 grid = make_grid(cam_frames, cols=grid_cols) + # to fit the text if only one cam available + if grid.shape[1] == 320: + cell_size = 640, 480 + grid = cv2.resize(grid, cell_size) + + cv2.putText(grid, f'Click on the web camera to use', (10, grid.shape[0] - 30), 0, 0.7, (200, 200, 200), 2) + cv2.namedWindow(window) cv2.setMouseCallback(window, mouse_callback, (cell_size, grid_cols, cam_frames)) cv2.imshow(window, grid) diff --git a/afy/predictor_local.py b/afy/predictor_local.py index 46f0aea19..2f7797ec9 100644 --- a/afy/predictor_local.py +++ b/afy/predictor_local.py @@ -6,7 +6,29 @@ from sync_batchnorm import DataParallelWithCallback import numpy as np import face_alignment -from animate import normalize_kp + + +def normalize_kp(kp_source, kp_driving, kp_driving_initial, adapt_movement_scale=False, + use_relative_movement=False, use_relative_jacobian=False): + if adapt_movement_scale: + source_area = ConvexHull(kp_source['value'][0].data.cpu().numpy()).volume + driving_area = ConvexHull(kp_driving_initial['value'][0].data.cpu().numpy()).volume + adapt_movement_scale = np.sqrt(source_area) / np.sqrt(driving_area) + else: + adapt_movement_scale = 1 + + kp_new = {k: v for k, v in kp_driving.items()} + + if use_relative_movement: + kp_value_diff = (kp_driving['value'] - kp_driving_initial['value']) + kp_value_diff *= adapt_movement_scale + kp_new['value'] = kp_value_diff + kp_source['value'] + + if use_relative_jacobian: + jacobian_diff = torch.matmul(kp_driving['jacobian'], torch.inverse(kp_driving_initial['jacobian'])) + kp_new['jacobian'] = torch.matmul(jacobian_diff, kp_source['jacobian']) + + return kp_new def to_tensor(a): @@ -31,7 +53,7 @@ def __init__(self, config_path, checkpoint_path, relative=False, adapt_movement_ def load_checkpoints(self): with open(self.config_path) as f: - config = yaml.load(f) + config = yaml.load(f, Loader=yaml.FullLoader) generator = OcclusionAwareGenerator(**config['model_params']['generator_params'], **config['model_params']['common_params']) diff --git a/afy/predictor_remote.py b/afy/predictor_remote.py index 1a2baec31..9c63a1f19 100644 --- a/afy/predictor_remote.py +++ b/afy/predictor_remote.py @@ -1,6 +1,6 @@ from arguments import opt from networking import SerializingContext, check_connection -from utils import Tee, TicToc, AccumDict, Once +from utils import Logger, TicToc, AccumDict, Once import multiprocessing as mp import queue @@ -25,7 +25,7 @@ def __init__(self, *args, in_addr=None, out_addr=None, **kwargs): self.out_addr = out_addr self.predictor_args = (args, kwargs) self.timing = AccumDict() - self.log = Tee('./var/log/predictor_remote.log') + self.log = Logger('./var/log/predictor_remote.log', verbose=opt.verbose) self.send_queue = mp.Queue(QUEUE_SIZE) self.recv_queue = mp.Queue(QUEUE_SIZE) @@ -88,8 +88,7 @@ def _send_recv_async(self, method, args, critical): 'id': self._i_msg } - if opt.verbose: - self.log("send", meta) + self.log("send", meta) if critical: self.send_queue.put((meta, data)) @@ -111,8 +110,7 @@ def _send_recv_async(self, method, args, critical): self.log('recv_queue is empty') return None - if opt.verbose: - self.log("recv", meta_recv) + self.log("recv", meta_recv) tt.tic() if meta_recv['name'] == 'predict': @@ -121,14 +119,15 @@ def _send_recv_async(self, method, args, critical): result = msgpack.unpackb(data_recv) self.timing.add('UNPACK', tt.toc()) - Once(self.timing, per=1) + if opt.verbose: + Once(self.timing, per=1) return result @staticmethod def send_worker(address, send_queue, worker_alive): timing = AccumDict() - log = Tee('./var/log/send_worker.log') + log = Logger('./var/log/send_worker.log', opt.verbose) ctx = SerializingContext() sender = ctx.socket(zmq.PUSH) @@ -149,7 +148,8 @@ def send_worker(address, send_queue, worker_alive): sender.send_data(*msg) timing.add('SEND', tt.toc()) - Once(timing, log, per=1) + if opt.verbose: + Once(timing, log, per=1) except KeyboardInterrupt: log("send_worker: user interrupt") finally: @@ -163,7 +163,7 @@ def send_worker(address, send_queue, worker_alive): @staticmethod def recv_worker(address, recv_queue, worker_alive): timing = AccumDict() - log = Tee('./var/log/recv_worker.log') + log = Logger('./var/log/recv_worker.log') ctx = SerializingContext() receiver = ctx.socket(zmq.PULL) @@ -189,7 +189,8 @@ def recv_worker(address, recv_queue, worker_alive): log('recv_queue full') continue - Once(timing, log, per=1) + if opt.verbose: + Once(timing, log, per=1) except KeyboardInterrupt: log("recv_worker: user interrupt") finally: diff --git a/afy/predictor_worker.py b/afy/predictor_worker.py index 2afca0460..05a959baa 100644 --- a/afy/predictor_worker.py +++ b/afy/predictor_worker.py @@ -1,7 +1,7 @@ from predictor_local import PredictorLocal from arguments import opt from networking import SerializingContext, check_connection -from utils import Tee, TicToc, AccumDict, Once +from utils import Logger, TicToc, AccumDict, Once import cv2 import numpy as np @@ -58,14 +58,14 @@ def run(self): @staticmethod def recv_worker(port, recv_queue, worker_alive): timing = AccumDict() - log = Tee('./var/log/recv_worker.log') + log = Logger('./var/log/recv_worker.log', verbose=opt.verbose) ctx = SerializingContext() socket = ctx.socket(zmq.PULL) socket.bind(f"tcp://*:{port}") socket.RCVTIMEO = RECV_TIMEOUT - log(f'Receiving on port {port}') + log(f'Receiving on port {port}', important=True) try: while worker_alive.value: @@ -92,17 +92,17 @@ def recv_worker(port, recv_queue, worker_alive): Once(timing, log, per=1) except KeyboardInterrupt: - log("recv_worker: user interrupt") + log("recv_worker: user interrupt", important=True) worker_alive.value = 0 - log("recv_worker exit") + log("recv_worker exit", important=True) @staticmethod def predictor_worker(recv_queue, send_queue, worker_alive): predictor = None predictor_args = () timing = AccumDict() - log = Tee('./var/log/predictor_worker.log') + log = Logger('./var/log/predictor_worker.log', verbose=opt.verbose) try: while worker_alive.value: @@ -129,7 +129,7 @@ def predictor_worker(recv_queue, send_queue, worker_alive): args = msgpack.unpackb(data) timing.add('UNPACK', tt.toc()) except ValueError: - log("Invalid Message") + log("Invalid Message", important=True) continue tt.tic() @@ -142,7 +142,7 @@ def predictor_worker(recv_queue, send_queue, worker_alive): del predictor predictor_args = args predictor = PredictorLocal(*predictor_args[0], **predictor_args[1]) - log("Initialized predictor with:", predictor_args) + log("Initialized predictor with:", predictor_args, important=True) result = True tt.tic() # don't account for init elif method['name'] == 'predict': @@ -172,24 +172,24 @@ def predictor_worker(recv_queue, send_queue, worker_alive): Once(timing, log, per=1) except KeyboardInterrupt: - log("predictor_worker: user interrupt") + log("predictor_worker: user interrupt", important=True) except Exception as e: - log("predictor_worker error") + log("predictor_worker error", important=True) traceback.print_exc() worker_alive.value = 0 - log("predictor_worker exit") + log("predictor_worker exit", important=True) @staticmethod def send_worker(port, send_queue, worker_alive): timing = AccumDict() - log = Tee('./var/log/send_worker.log') + log = Logger('./var/log/send_worker.log', verbose=opt.verbose) ctx = SerializingContext() socket = ctx.socket(zmq.PUSH) socket.bind(f"tcp://*:{port}") - log(f'Sending on port {port}') + log(f'Sending on port {port}', important=True) try: while worker_alive.value: @@ -215,10 +215,10 @@ def send_worker(port, send_queue, worker_alive): Once(timing, log, per=1) except KeyboardInterrupt: - log("predictor_worker: user interrupt") + log("predictor_worker: user interrupt", important=True) worker_alive.value = 0 - log("send_worker exit") + log("send_worker exit", important=True) def run_worker(in_port=None, out_port=None): diff --git a/afy/utils.py b/afy/utils.py index 8b8741670..6c66e59f9 100644 --- a/afy/utils.py +++ b/afy/utils.py @@ -34,6 +34,18 @@ def flush(self): self.file.flush() +class Logger(): + def __init__(self, filename, verbose=True): + self.tee = Tee(filename) + self.verbose = verbose + + def __call__(self, *args, important=False, **kwargs): + if not self.verbose and not important: + return + + self.tee(*args, **kwargs) + + class Once(): _id = {} @@ -70,13 +82,6 @@ def tocp(self, str): log(f"{str} took {t:.4f}ms") return t - @staticmethod - def print(name=''): - log(f'\n=== {name} Timimg ===') - for fn, times in self.timing.items(): - min, max, mean, p95 = np.min(times), np.max(times), np.mean(times), np.percentile(times, 95) - log(f'{fn}:\tmin: {min:.4f}\tmax: {max:.4f}\tmean: {mean:.4f}ms\tp95: {p95:.4f}ms') - class AccumDict: def __init__(self, num_f=3): @@ -109,6 +114,10 @@ def __repr__(self): return self.__str__() +def clamp(value, min_value, max_value): + return max(min(value, max_value), min_value) + + def crop(img, p=0.7, offset_x=0, offset_y=0): h, w = img.shape[:2] x = int(min(w, h) * p) @@ -116,12 +125,16 @@ def crop(img, p=0.7, offset_x=0, offset_y=0): r = w - l u = (h - x) // 2 d = h - u + + offset_x = clamp(offset_x, -l, w - r) + offset_y = clamp(offset_y, -u, h - d) + l += offset_x r += offset_x u += offset_y d += offset_y - return img[u:d, l:r], (l,r,u,d,w,h) + return img[u:d, l:r], (offset_x, offset_y) def pad_img(img, target_size, default_pad=0): diff --git a/afy/videocaptureasync.py b/afy/videocaptureasync.py index 16abd2654..141822b21 100644 --- a/afy/videocaptureasync.py +++ b/afy/videocaptureasync.py @@ -15,7 +15,7 @@ def __init__(self, src=0, width=640, height=480): self.cap = cv2.VideoCapture(self.src) if not self.cap.isOpened(): raise RuntimeError("Cannot open camera") - + self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) self.grabbed, self.frame = self.cap.read() @@ -36,7 +36,7 @@ def start(self): self.thread = threading.Thread(target=self.update, args=(), daemon=True) self.thread.start() - # (warmup) wait for the first successfully grabbed frame + # (warmup) wait for the first successfully grabbed frame warmup_start_time = time.time() while not self.grabbed: warmup_elapsed_time = (time.time() - warmup_start_time) @@ -44,20 +44,24 @@ def start(self): raise RuntimeError(f"Failed to succesfully grab frame from the camera (timeout={WARMUP_TIMEOUT}s). Try to restart.") time.sleep(0.5) - + return self def update(self): while self.started: grabbed, frame = self.cap.read() + if not grabbed or frame is None or frame.size == 0: + continue with self.read_lock: self.grabbed = grabbed self.frame = frame def read(self): - with self.read_lock: - frame = self.frame.copy() - grabbed = self.grabbed + while True: + with self.read_lock: + frame = self.frame.copy() + grabbed = self.grabbed + break return grabbed, frame def stop(self): diff --git a/avatarify.ipynb b/avatarify.ipynb index 826d50ce3..7b15d83b7 100644 --- a/avatarify.ipynb +++ b/avatarify.ipynb @@ -419,6 +419,7 @@ "source": [ "config =\\\n", "f\"\"\"\n", + "version: 2\n", "authtoken: {authtoken}\n", "region: {region}\n", "console_ui: False\n", @@ -649,4 +650,4 @@ "outputs": [] } ] -} \ No newline at end of file +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..cafdab79d --- /dev/null +++ b/docs/README.md @@ -0,0 +1,322 @@ +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) + +:arrow_forward: [Demo](https://youtu.be/Q7LFDT-FRzs) + +:arrow_forward: [AI-generated Elon Musk](https://youtu.be/lONuXGNqLO0) + +## Table of Contents +- [Requirements](#requirements) +- [Install](#install) + - [Download network weights](#download-network-weights) + - [Linux](#linux) + - [Mac](#mac) + - [Windows](#windows) + - [Remote GPU](#remote-gpu) + - [Docker](#docker) +- [Setup avatars](#setup-avatars) +- [Run](#run) + - [Linux](#linux-1) + - [Mac](#mac-1) + - [Windows](#windows-1) +- [Controls](#controls) +- [Driving your avatar](#driving-your-avatar) +- [Configure video meeting app](#configure-video-meeting-app) + - [Skype](#skype) + - [Zoom](#zoom) + - [Teams](#teams) + - [Slack](#slack) +- [Uninstall](#uninstall) +- [Contribution](#contribution) +- [FAQ](#faq) +- [Troubleshooting](#troubleshooting) + +## Requirements + +You can run Avatarify in two modes: *locally* and *remotely*. + +To run Avatarify *locally* you need a CUDA-enabled (NVIDIA) video card. Otherwise it will fallback to the central processor and run very slowly. These are performance metrics for some hardware: + +- GeForce GTX 1080 Ti: **33 frames per second** +- GeForce GTX 1070: **15 frames per second** +- GeForce GTX 950: **9 frames per second** + +You can also run Avatarify *remotely* on [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) (easy) or on a [dedicated server](https://github.com/alievk/avatarify-python/wiki/Remote-GPU) with a GPU (harder). There are no special PC requirements for this mode, only a stable internet connection. + +Of course, you also need a webcam! + + + +## Install + +#### Download network weights +Download model's weights from [here](https://openavatarify.s3-avatarify.com/weights/vox-adv-cpk.pth.tar) or [here](https://yadi.sk/d/M0FWpz2ExBfgAA) or [here](https://drive.google.com/file/d/1coUCdyRXDbpWnEkA99NLNY60mb9dQ_n3/view?usp=sharing) [228 MB, md5sum `8a45a24037871c045fbb8a6a8aa95ebc`] + +#### Linux +Linux uses `v4l2loopback` to create virtual camera. + + +1. Download [Miniconda Python 3.7](https://docs.conda.io/en/latest/miniconda.html#linux-installers) and install using command: +```bash +bash Miniconda3-latest-Linux-x86_64.sh +``` +2. Clone `avatarify` and install its dependencies (sudo privelege is required): +```bash +git clone https://github.com/alievk/avatarify-python.git +cd avatarify-python +bash scripts/install.sh +``` +3. [Download network weights](#download-network-weights) and place `vox-adv-cpk.pth.tar` file in the `avatarify-python` directory (don't unpack it). + + +#### Mac + + + +We will use [CamTwist](http://camtwiststudio.com) to create virtual camera for Mac. + +1. Install [Miniconda Python 3.7](https://docs.conda.io/en/latest/miniconda.html#macosx-installers) or use *Homebrew Cask*: `brew install --cask miniconda`. +2. [Download](https://github.com/alievk/avatarify-python/archive/master.zip) and unpack the repository or use `git`: +```bash +git clone https://github.com/alievk/avatarify-python.git +cd avatarify-python +bash scripts/install_mac.sh +``` +3. Download and install [CamTwist](http://camtwiststudio.com) from [here](http://camtwiststudio.com/download). It's easy. + +#### Windows + + + +:arrow_forward: [Video tutorial](https://youtu.be/lym9ANVb120) + +This guide is tested for Windows 10. + + +1. Install [Miniconda Python 3.8](https://docs.conda.io/en/latest/miniconda.html#windows-installers). +2. Install [Git](https://git-scm.com/download/win). +3. Press Windows button and type "miniconda". Run suggested Anaconda Prompt. +4. Download and install Avatarify (please copy-paste these commands and don't change them): +```bash +git clone https://github.com/alievk/avatarify-python.git +cd avatarify-python +scripts\install_windows.bat +``` +5. [Download network weights](#download-network-weights) and place `vox-adv-cpk.pth.tar` file in the `avatarify-python` directory (don't unpack it). +6. Run `run_windows.bat`. If installation was successful, two windows "cam" and "avatarify" will appear. Leave these windows open for the next installation steps. +7. Install [OBS Studio](https://obsproject.com/) for capturing Avatarify output. +8. Install [VirtualCam plugin](https://obsproject.com/forum/resources/obs-virtualcam.539/). Choose `Install and register only 1 virtual camera`. +9. Run OBS Studio. +10. In the Sources section, press on Add button ("+" sign), select Windows Capture and press OK. In the appeared window, choose "[python.exe]: avatarify" in Window drop-down menu and press OK. Then select Edit -> Transform -> Fit to screen. +11. In OBS Studio, go to Tools -> VirtualCam. Check AutoStart, set Buffered Frames to 0 and press Start. +12. Now `OBS-Camera` camera should be available in Zoom (or other videoconferencing software). + +The steps 10-11 are required only once during setup. + +#### Remote GPU + +You can offload the heavy work to [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) or a [server with a GPU](https://github.com/alievk/avatarify-python/wiki/Remote-GPU) and use your laptop just to communicate the video stream. The server and client software are native and dockerized available. + +### Docker +Docker images are only availabe on Linux. + +1. Install Docker following the [Documentation](https://docs.docker.com/engine/install/). Then run this [step](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user) to make docker available for your user. +2. For using the gpu (hardly recommended): Install nvidia drivers and [nvidia docker](https://github.com/NVIDIA/nvidia-docker#quickstart). +3. Clone `avatarify-python` and install its dependencies (v4l2loopback kernel module): +```bash +git clone https://github.com/alievk/avatarify-python.git +cd avatarify-python +bash scripts/install_docker.sh +``` +4. Build the Dockerfile: +```bash +cd avatarify-python +docker build -t avatarify . +``` +## Setup avatars +Avatarify comes with a standard set of avatars of famous people, but you can extend this set simply copying your avatars into `avatars` folder. + +Follow these advices for better visual quality: +* Make square crop of your avatar picture. +* Crop avatar's face so that it's not too close not too far. Use standard avatars as reference. +* Prefer pictures with uniform background. It will diminish visual artifacts. + +## Run +Your web cam must be plugged-in. + +**Note:** run your video-conferencing app only after Avatarify is started. + +#### Linux +The run script will create virtual camera `/dev/video9`. You can change these settings in `scripts/settings.sh`. + + +You can use command `v4l2-ctl --list-devices` to list all devices in your system. + +Run: +```bash +bash run.sh +``` +If you haven't installed a GPU add the `--no-gpus` flag. In order to use Docker add the `--docker` flag. + +`cam` and `avatarify` windows will pop-up. The `cam` window is for controlling your face position and `avatarify` is for the avatar animation preview. Please follow these [recommendations](#driving-your-avatar) to drive your avatars. + +#### Mac +*Note*: On Mac Avatarify runs only with [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) or a [dedicated server](https://github.com/alievk/avatarify-python/wiki/Remote-GPU) with GPU. + +Please find where you downloaded `avatarify` and substitute path `/path/to/avatarify` below. + + +1. To run Avatarify please follow instructions for [Google Colab](https://colab.research.google.com/github/alievk/avatarify/blob/master/avatarify.ipynb) or a [dedicated server](https://github.com/alievk/avatarify-python/wiki/Remote-GPU). +2. Go to [CamTwist](http://camtwiststudio.com). +3. Choose `Desktop+` and press `Select`. +4. In the `Settings` section choose `Confine to Application Window` and select `python (avatarify)` from the drop-down menu. + +`cam` and `avatarify` windows will pop-up. The `cam` window is for controlling your face position and `avatarify` is for the avatar animation preview. Please follow these [recommendations](#driving-your-avatar) to drive your avatars. + +#### Windows + + + +1. In Anaconda Prompt: +``` +cd C:\path\to\avatarify +run_windows.bat +``` +2. Run OBS Studio. It should automaitcally start streaming video from Avatarify to `OBS-Camera`. + +`cam` and `avatarify` windows will pop-up. The `cam` window is for controlling your face position and `avatarify` is for the avatar animation preview. Please follow these [recommendations](#driving-your-avatar) to drive your avatars. + +**Note:** To reduce video latency, in OBS Studio right click on the preview window and uncheck Enable Preview. + +## Controls + + +Keys | Controls +--- | --- +1-9 | These will immediately switch between the first 9 avatars. +Q | Turns on StyleGAN-generated avatar. Every time you push the button – new avatar is sampled. +0 | Toggles avatar display on and off. +A/D | Previous/next avatar in folder. +W/S | Zoom camera in/out. +U/H/J/K | Translate camera. `H` - left, `K` - right, `U` - up, `J` - Down by 5 pixels. Add `Shift` to adjust by 1 pixel. +Shift-Z | Reset camera zoom and translation +Z/C | Adjust avatar target overlay opacity. +X | Reset reference frame. +F | Toggle reference frame search mode. +R | Mirror reference window. +T | Mirror output window. +L | Reload avatars. +I | Show FPS +O | Toggle face detection overlay. +ESC | Quit + +## Driving your avatar + +These are the main principles for driving your avatar: + +* Align your face in the camera window as closely as possible in proportion and position to the target avatar. Use zoom in/out function (W/S keys) and camera left, right, up, down translation (U/H/J/K keys). When you have aligned, hit 'X' to use this frame as reference to drive the rest of the animation +* Use the image overlay function (Z/C keys) or the face detection overlay function (O key) to match your and avatar's face expressions as close as possible + +Alternatively, you can hit 'F' for the software to attempt to find a better reference frame itself. This will slow down the framerate, but while this is happening, you can keep moving your head around: the preview window will flash green when it finds your facial pose is a closer match to the avatar than the one it is currently using. You will see two numbers displayed as well: the first number is how closely you are currently aligned to the avatar, and the second number is how closely the reference frame is aligned. + +You want to get the first number as small as possible - around 10 is usually a good alignment. When you are done, press 'F' again to exit reference frame search mode. + +You don't need to be exact, and some other configurations can yield better results still, but it's usually a good starting point. + +## Configure video meeting app + +Avatarify supports any video-conferencing app where video input source can be changed (Zoom, Skype, Hangouts, Slack, ...). Here are a few examples how to configure particular app to use Avatarify. + +### Skype + +Go to Settings -> Audio & Video, choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) camera. + + + +### Zoom + +Go to Settings -> Video and choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) from Camera drop-down menu. + + + +### Teams + +Go to your profile picture -> Settings -> Devices and choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) from Camera drop-down menu. + + + +### Slack + +Make a call, allow browser using cameras, click on Settings icon, choose `avatarify` (Linux), `CamTwist` (Mac) or `OBS-Camera` (Windows) in Video settings drop-down menu. + + + + +## Uninstall +To remove Avatarify and its related programs follow the [instructions](https://github.com/alievk/avatarify-python/wiki/Removing-Avatarify) in the Wiki. + + +## Contribution + +Our goal is to democratize photorealistic avatars for video-conferencing. To make the technology even more accessible, we have to tackle the following problems: + +* ~~Add support for more platforms (Linux and Mac are already supported).~~ +* ~~Remote GPU support. This is a work in progress.~~ +* Porting to non-CUDA GPUs (Intel integrated GPUs, AMD GPUs, etc) and optimization. The goal is to run Avatarify real-time (at least 10FPS) on modern laptops. + +Please make pull requests if you have any improvements or bug-fixes. + + +## FAQ + +Q: **Do I need any knowledge of programming to run Avatarify?** +A: Not really, but you need some beginner-level knowledge of the command line. For Windows we recorded a video [tutorial](https://www.youtube.com/watch?v=lym9ANVb120), so it’ll be easy to install. + +Q: **Why does it work so slow on my Macbook?** +A: The model used in Avatarify requires a CUDA-enabled NVIDIA GPU to perform heavy computations. Macbooks don’t have such GPUs, and for processing use CPU, which has much less computing power to run Avatarify smoothly. + +Q: **I don’t have a NVIDIA GPU, can I run it?** +A: You still can run it without a NVIDIA GPU, but with drastically reduced performance (<1fps). + +Q: **I have an ATI GPU (e.g. Radeon). Why does it work so slow?** +A: To run the neural network Avatarify uses PyTorch library, which is optimized for CUDA. If PyTorch can’t find a CUDA-enabled GPU in your system it will fallback to CPU. The performance on the CPU will be much worse. + +Q: **How to add a new avatar?** +A: It’s easy. All you need is to find a picture of your avatar and put it in the `avatars` folder. [More](https://github.com/alievk/avatarify-python#setup-avatars). + +Q: **My avatar looks distorted.** +A: You need to calibrate your face position. Please follow the [tips](https://github.com/alievk/avatarify-python#driving-your-avatar) or watch the video [tutorial](https://youtu.be/lym9ANVb120?t=662). + +Q: **Can I use a cloud GPU?** +A: This is work in progress. See the relevant [discussion](https://github.com/alievk/avatarify-python/issues/115). + +Q: **Avatarify crashed, what to do?** +A: First, try to find your error in the [troubleshooting](https://github.com/alievk/avatarify-python#troubleshooting) section. If it is not there, try to find it in the [issues](https://github.com/alievk/avatarify-python/issues). If you couldn’t find your issue there, please open a new one using the issue template. + +Q: **Can I use Avatarify for commercial purposes?** +A: No. Avatarify and First Order Motion Model are licensed under Creative Commons Non-Commercial license, which prohibits commercial use. + +Q: **What video conferencing apps does Avatarify support?** +A: Avatarify creates a virtual camera which can be plugged into any app where video input source can be changed (Zoom, Skype, Hangouts, Slack, ...). + +Q: **Where can I discuss Avatarify-related topics with the community?** +A: We have Slack. Please join: [](https://join.slack.com/t/avatarify/shared_invite/zt-dyoqy8tc-~4U2ObQ6WoxuwSaWKKVOgg) + + +## Troubleshooting + +Please follow the [Wiki](https://github.com/alievk/avatarify-python/wiki/Troubleshooting) page. + diff --git a/docs/appstore-badge.png b/docs/appstore-badge.png new file mode 100644 index 000000000..8e91eb3bd Binary files /dev/null and b/docs/appstore-badge.png differ diff --git a/docs/google-play-badge.png b/docs/google-play-badge.png new file mode 100644 index 000000000..1eaec7543 Binary files /dev/null and b/docs/google-play-badge.png differ diff --git a/requirements.txt b/requirements.txt index ad4c21da1..1e6240df0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,33 +1,7 @@ -#cffi==1.11.5 -cloudpickle==0.5.3 -cycler==0.10.0 -dask==0.18.2 -decorator==4.3.0 -imageio==2.3.0 -kiwisolver==1.0.1 -matplotlib==2.2.2 -networkx==2.1 -numpy==1.15.0 -pandas==0.23.4 -Pillow==6.2.2 -pycparser==2.18 -pygit==0.1 -pyparsing==2.2.0 -python-dateutil==2.7.3 -pytz==2018.5 -PyWavelets==0.5.2 -PyYAML==5.1 -scikit-image==0.14.0 -scikit-learn==0.19.2 -scipy==1.1.0 -six==1.11.0 -toolz==0.9.0 -#torch==1.0.0 -#torchvision==0.2.1 -tqdm==4.24.0 -requests==2.23 -face-alignment==1.0.0 -pyfakewebcam==0.1.0 -msgpack-numpy==0.4.5 -pyzmq==19.0.0 -#blosc==1.7.0 +opencv-python==4.2.0.34 +face-alignment==1.3.3 +pyzmq==20.0.0 +msgpack-numpy==0.4.7.1 +pyyaml==5.4 +requests==2.32.0 +pyfakewebcam==0.1.0 \ No newline at end of file diff --git a/requirements_client.txt b/requirements_client.txt index 0d3028df7..da8920832 100644 --- a/requirements_client.txt +++ b/requirements_client.txt @@ -1,6 +1,6 @@ numpy==1.15.0 PyYAML==5.1 -requests==2.23 +requests==2.32.0 msgpack-numpy==0.4.5 pyzmq==19.0.0 -opencv-python==3.4.9.33 \ No newline at end of file +opencv-python==3.4.5.20 diff --git a/run.sh b/run.sh index d56fd2d4b..24ffc6ed3 100755 --- a/run.sh +++ b/run.sh @@ -4,11 +4,18 @@ ENABLE_CONDA=1 ENABLE_VCAM=1 +KILL_PS=1 +USE_DOCKER=0 +IS_WORKER=0 +IS_CLIENT=0 +DOCKER_IS_LOCAL_CLIENT=0 +DOCKER_NO_GPU=0 FOMM_CONFIG=fomm/config/vox-adv-256.yaml FOMM_CKPT=vox-adv-cpk.pth.tar ARGS="" +DOCKER_ARGS="" while (( "$#" )); do case "$1" in @@ -25,6 +32,31 @@ while (( "$#" )); do KILL_PS=0 shift ;; + --docker) + USE_DOCKER=1 + shift + ;; + --no-gpus) + DOCKER_NO_GPU=1 + shift + ;; + --is-worker) + IS_WORKER=1 + ARGS="$ARGS $1" + DOCKER_ARGS="$DOCKER_ARGS -p 5557:5557 -p 5558:5558" + shift + ;; + --is-client) + IS_CLIENT=1 + ARGS="$ARGS $1" + shift + ;; + --is-local-client) + IS_CLIENT=1 + DOCKER_IS_LOCAL_CLIENT=1 + ARGS="$ARGS --is-client" + shift + ;; *|-*|--*) ARGS="$ARGS $1" shift @@ -34,28 +66,86 @@ done eval set -- "$ARGS" -if [[ $KILL_PS == 1 ]]; then - kill -9 $(ps aux | grep 'afy/cam_fomm.py' | awk '{print $2}') 2> /dev/null -fi -source scripts/settings.sh -if [[ $ENABLE_VCAM == 1 ]]; then - bash scripts/create_virtual_camera.sh -fi +if [[ $USE_DOCKER == 0 ]]; then + + if [[ $KILL_PS == 1 ]]; then + kill -9 $(ps aux | grep 'afy/cam_fomm.py' | awk '{print $2}') 2> /dev/null + fi + + source scripts/settings.sh + + if [[ $ENABLE_VCAM == 1 ]]; then + bash scripts/create_virtual_camera.sh + fi + + if [[ $ENABLE_CONDA == 1 ]]; then + source $(conda info --base)/etc/profile.d/conda.sh + conda activate $CONDA_ENV_NAME + fi + + export PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/fomm + + python afy/cam_fomm.py \ + --config $FOMM_CONFIG \ + --checkpoint $FOMM_CKPT \ + --virt-cam $CAMID_VIRT \ + --relative \ + --adapt_scale \ + $@ +else -if [[ $ENABLE_CONDA == 1 ]]; then - source $(conda info --base)/etc/profile.d/conda.sh - conda activate $CONDA_ENV_NAME -fi + source scripts/settings.sh -export PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/fomm + if [[ $ENABLE_VCAM == 1 ]]; then + bash scripts/create_virtual_camera.sh + fi + + if [[ $DOCKER_NO_GPU == 0 ]]; then + if nvidia-container-runtime -v &> /dev/null; then + DOCKER_ARGS="$DOCKER_ARGS --runtime=nvidia" + echo "Warning : Outdated Docker gpu support, please update !" + else + DOCKER_ARGS="$DOCKER_ARGS --gpus all" + fi + fi -python afy/cam_fomm.py \ - --config $FOMM_CONFIG \ - --checkpoint $FOMM_CKPT \ - --virt-cam $CAMID_VIRT \ - --relative \ - --adapt_scale \ - $@ + if [[ $DOCKER_IS_LOCAL_CLIENT == 1 ]]; then + DOCKER_ARGS="$DOCKER_ARGS --network=host" + elif [[ $IS_CLIENT == 1 ]]; then + DOCKER_ARGS="$DOCKER_ARGS -p 5557:5554 -p 5557:5558" + fi + + if [[ $IS_WORKER == 0 ]]; then + xhost +local:root + docker run $DOCKER_ARGS -it --rm --privileged \ + -v $PWD:/root/.torch/models \ + -v $PWD/avatars:/app/avatarify/avatars \ + --env="DISPLAY" \ + --env="QT_X11_NO_MITSHM=1" \ + --volume="/tmp/.X11-unix:/tmp/.X11-unix:rw" \ + avatarify python3 afy/cam_fomm.py \ + --config $FOMM_CONFIG \ + --checkpoint $FOMM_CKPT \ + --virt-cam $CAMID_VIRT \ + --relative \ + --adapt_scale \ + $@ + xhost -local:root + + else + docker run $DOCKER_ARGS -it --rm --privileged \ + -v $PWD:/root/.torch/models \ + -v $PWD/avatars:/app/avatarify/avatars \ + avatarify python3 afy/cam_fomm.py \ + --config $FOMM_CONFIG \ + --checkpoint $FOMM_CKPT \ + --virt-cam $CAMID_VIRT \ + --relative \ + --adapt_scale \ + $@ + fi + +fi diff --git a/scripts/download_data.sh b/scripts/download_data.sh index 150dce5e5..a65f588df 100755 --- a/scripts/download_data.sh +++ b/scripts/download_data.sh @@ -1,15 +1,14 @@ #!/usr/bin/env bash -# vox-adv-cpk-wget.pth.tar (https://drive.google.com/file/d/1coUCdyRXDbpWnEkA99NLNY60mb9dQ_n3/view) -echo "Downloading model's weights (vox-adv-cpk.pth.tar)" - -file_id=1coUCdyRXDbpWnEkA99NLNY60mb9dQ_n3 filename=vox-adv-cpk.pth.tar -echo "Getting cookie" -curl -sc /tmp/cookie "https://drive.google.com/uc?export=download&id=${file_id}" > /dev/null -code="$(awk '/_warning_/ {print $NF}' /tmp/cookie)" -echo "Downloading data" -curl -Lb /tmp/cookie "https://drive.google.com/uc?export=download&confirm=${code}&id=${file_id}" -o ${filename} + +curl https://openavatarify.s3-avatarify.com/weights/$filename -o $filename echo "Expected checksum: 8a45a24037871c045fbb8a6a8aa95ebc" -echo "Found checksum: $(md5sum ${filename})" + +if [ "$(uname)" == "Darwin" ]; then + sum=`md5 ${filename}` +else + sum=`md5sum ${filename}` +fi +echo "Found checksum: $sum" diff --git a/scripts/get_ngrok.sh b/scripts/get_ngrok.sh index 99cf01d6e..9a14b722b 100755 --- a/scripts/get_ngrok.sh +++ b/scripts/get_ngrok.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -command -v ./ngrok >/dev/null 2>&1 +# Check if ngrok is installed +command -v ngrok >/dev/null 2>&1 if [[ $? -ne 0 ]]; then - echo ngrok is not found, installing... - wget -q -nc https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip - unzip -qq -n ngrok-stable-linux-amd64.zip - echo Done! + echo "ngrok is not found, installing..." + wget -q -nc https://bin.equinox.io/c/bNyj1mQVY4c/ngrok-v3-stable-linux-amd64.tgz + tar -xzf ngrok-v3-stable-linux-amd64.tgz + echo "Done!" fi - diff --git a/scripts/install.sh b/scripts/install.sh index f3b994500..58f2b0f32 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -21,8 +21,8 @@ source $(conda info --base)/etc/profile.d/conda.sh conda create -y -n $CONDA_ENV_NAME python=3.7 conda activate $CONDA_ENV_NAME -conda install -y pytorch==1.0.0 torchvision==0.2.1 cuda100 -c pytorch -conda install -y python-blosc==1.7.0 -c conda-forge +conda install -y numpy==1.19.0 scikit-image python-blosc==1.7.0 -c conda-forge +conda install -y pytorch==1.7.1 torchvision cudatoolkit=11.0 -c pytorch # FOMM rm -rf fomm 2> /dev/null diff --git a/scripts/install_docker.sh b/scripts/install_docker.sh new file mode 100644 index 000000000..b4e3802eb --- /dev/null +++ b/scripts/install_docker.sh @@ -0,0 +1,9 @@ +if [[ ! $@ =~ "no-vcam" ]]; then + rm -rf v4l2loopback 2> /dev/null + git clone https://github.com/umlaeute/v4l2loopback + echo "--- Installing v4l2loopback (sudo privelege required)" + cd v4l2loopback + make && sudo make install + sudo depmod -a + cd .. +fi diff --git a/scripts/install_windows.bat b/scripts/install_windows.bat index f8dd1d27d..be90396da 100644 --- a/scripts/install_windows.bat +++ b/scripts/install_windows.bat @@ -2,18 +2,19 @@ REM Check prerequisites call conda --version >nul 2>&1 && ( echo conda found ) || ( echo conda not found. Please refer to the README and install Miniconda. && exit /B 1) -call git --version >nul 2>&1 && ( echo git found ) || ( echo git not found. Please refer to the README and install Git. && exit /B 1) +REM call git --version >nul 2>&1 && ( echo git found ) || ( echo git not found. Please refer to the README and install Git. && exit /B 1) call scripts/settings_windows.bat call conda create -y -n %CONDA_ENV_NAME% python=3.7 call conda activate %CONDA_ENV_NAME% -call conda install -y pytorch==1.0.0 torchvision==0.2.1 cuda100 -c pytorch -call conda install -y python-blosc==1.7.0 -c conda-forge +call conda install -y numpy==1.19.0 scikit-image python-blosc==1.7.0 -c conda-forge +call conda install -y pytorch==1.7.1 torchvision cudatoolkit=11.0 -c pytorch +call conda install -y -c anaconda git REM ###FOMM### call rmdir fomm /s /q call git clone https://github.com/alievk/first-order-model.git fomm -call pip install -r requirements.txt +call pip install -r requirements.txt --use-feature=2020-resolver