國慶出遊神器：魔幻黑科技換天造物，讓vlog秒變科幻大片！
阿新 • • 發佈：2021-09-30
摘要：國慶旅遊景點人太多，拍出來的照片全是人人人、車車車，該怎麼辦？不妨試試這個黑科技，讓你的出遊vlog秒變科幻大片。
本文分享自華為雲社群《國慶出遊神器，魔幻黑科技換天造物，讓vlog秒變科幻大片！》，作者：技術火炬手 。
國慶出遊，無論是拍人、拍景或是其他，“天空”都是關鍵元素。比如，一張平平無奇的景物圖加上落日餘暉的天空色調，氛圍感就有了。
當然，自然景觀的天空還不是最酷炫的。今天給大家介紹一款基於原生視訊的AI處理方法，不僅可以一鍵置換天空背景，還可以打造任意“天空之城”。
比如換成《星際迷航》中的浩瀚星空、宇宙飛船，將自己隨手拍的平平無奇vlog秒變為科幻大片，畫面毫無違和感。
該方法源自Github上的開源專案SkyAR，它可以自動識別天空，然後將天空從圖片中切割出來，再將天空替換成目標天空，從而實現魔法換天。
下面，我們將基於SkyAR和ModelArts的JupyterLab從零開始“換天造物”。只要腦洞夠大，利用這項AI技術，就可以創造出無限種玩法。
本案例在CPU和GPU下面均可執行，CPU環境執行預計花費9分鐘，GPU環境執行預計花費2分鐘。
實驗目標
通過本案例的學習：
瞭解影象分割的基本應用；
瞭解運動估計的基本應用；
瞭解影象混合的基本應用。
注意事項
- 如果您是第一次使用 JupyterLab，請檢視《ModelArts JupyterLab使用指導》瞭解使用方法；
- 如果您在使用 JupyterLab 過程中碰到報錯，請參考《ModelArts JupyterLab常見問題解決辦法》嘗試解決問題。
實驗步驟
1、安裝和匯入依賴包
import os import moxing as mox file_name = 'SkyAR' if not os.path.exists(file_name): mox.file.copy('obs://modelarts-labs-bj4-v2/case_zoo/SkyAR/SkyAR.zip', 'SkyAR.zip') os.system('unzip SkyAR.zip') os.system('rm SkyAR.zip') mox.file.copy_parallel('obs://modelarts-labs-bj4-v2/case_zoo/SkyAR/resnet50-19c8e357.pth', '/home/ma-user/.cache/torch/checkpoints/resnet50-19c8e357.pth') INFO:root:Using MoXing-v1.17.3-43fbf97f INFO:root:Using OBS-Python-SDK-3.20.7 !pip uninstall opencv-python -y !pip uninstall opencv-contrib-python -y Found existing installation: opencv-python 4.1.2.30 Uninstalling opencv-python-4.1.2.30: Successfully uninstalled opencv-python-4.1.2.30 WARNING: Skipping opencv-contrib-python as it is not installed. !pip install opencv-contrib-python==4.5.3.56 Looking in indexes: http://repo.myhuaweicloud.com/repository/pypi/simple Collecting opencv-contrib-python==4.5.3.56 Downloading http://repo.myhuaweicloud.com/repository/pypi/packages/3f/ce/36772cc6d9061b423b080e86919fd62cdef0837263f29ba6ff92e07f72d7/opencv_contrib_python-4.5.3.56-cp37-cp37m-manylinux2014_x86_64.whl (56.1 MB) |████████████████████████████████| 56.1 MB 166 kB/s eta 0:00:01|█████▋ | 9.8 MB 9.4 MB/s eta 0:00:05 MB 9.4 MB/s eta 0:00:05███▏ | 26.6 MB 9.4 MB/s eta 0:00:04/s eta 0:00:03��██▍ | 35.8 MB 9.4 MB/s eta 0:00:03�███████████▌ | 42.9 MB 9.4 MB/s eta 0:00:02��██████████████▎ | 49.6 MB 166 kB/s eta 0:00:40 Requirement already satisfied: numpy>=1.14.5 in /home/ma-user/anaconda3/envs/PyTorch-1.4/lib/python3.7/site-packages (from opencv-contrib-python==4.5.3.56) (1.20.3) Installing collected packages: opencv-contrib-python Successfully installed opencv-contrib-python-4.5.3.56 WARNING: You are using pip version 20.3.3; however, version 21.1.3 is available. You should consider upgrading via the '/home/ma-user/anaconda3/envs/PyTorch-1.4/bin/python -m pip install --upgrade pip' command. cd SkyAR/ /home/ma-user/work/Untitled Folder/SkyAR import time import json import base64 import numpy as np import matplotlib.pyplot as plt import cv2 import argparse from networks import * from skyboxengine import * import utils import torch from IPython.display import clear_output, Image, display, HTML %matplotlib inline # 如果存在GPU則在GPU上面執行 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") INFO:matplotlib.font_manager:generated new fontManager
2、預覽一下原視訊
video_name = "test_videos/sky.mp4" def arrayShow(img): img = cv2.resize(img, (0, 0), fx=0.25, fy=0.25, interpolation=cv2.INTER_NEAREST) _,ret = cv2.imencode('.jpg', img) return Image(data=ret) # 開啟一個視訊流 cap = cv2.VideoCapture(video_name) frame_id = 0 while True: try: clear_output(wait=True) # 清除之前的顯示 ret, frame = cap.read() # 讀取一幀圖片 if ret: frame_id += 1 if frame_id > 200: break cv2.putText(frame, str(frame_id), (5, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1) # 畫frame_id tmp = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # 轉換色彩模式 img = arrayShow(frame) display(img) # 顯示圖片 time.sleep(0.05) # 執行緒睡眠一段時間再處理下一幀圖片 else: break except KeyboardInterrupt: cap.release() cap.release()
3、預覽一下要替換的天空圖片
img= cv2.imread('skybox/sky.jpg') img2 = img[:,:,::-1] plt.imshow(img2) <matplotlib.image.AxesImage at 0x7fbea986c590>
4、自定義訓練引數
可以根據自己的需要, 修改下面的引數
skybox_center_crop: 天空體中心偏移
auto_light_matching: 自動亮度匹配
relighting_factor: 補光
recoloring_factor: 重新著色
halo_effect: 光環效應
parameter = { "net_G": "coord_resnet50", "ckptdir": "./checkpoints_G_coord_resnet50", "input_mode": "video", "datadir": "./test_videos/sky.mp4", "skybox": "sky.jpg", "in_size_w": 384, "in_size_h": 384, "out_size_w": 845, "out_size_h": 480, "skybox_center_crop": 0.5, "auto_light_matching": False, "relighting_factor": 0.8, "recoloring_factor": 0.5, "halo_effect": True, "output_dir": "./jpg_output", "save_jpgs": False } str_json = json.dumps(parameter) class Struct: def __init__(self, **entries): self.__dict__.update(entries) def parse_config(): data = json.loads(str_json) args = Struct(**data) return args args = parse_config() class SkyFilter(): def __init__(self, args): self.ckptdir = args.ckptdir self.datadir = args.datadir self.input_mode = args.input_mode self.in_size_w, self.in_size_h = args.in_size_w, args.in_size_h self.out_size_w, self.out_size_h = args.out_size_w, args.out_size_h self.skyboxengine = SkyBox(args) self.net_G = define_G(input_nc=3, output_nc=1, ngf=64, netG=args.net_G).to(device) self.load_model() self.video_writer = cv2.VideoWriter('out.avi', cv2.VideoWriter_fourcc(*'MJPG'), 20.0, (args.out_size_w, args.out_size_h)) self.video_writer_cat = cv2.VideoWriter('compare.avi', cv2.VideoWriter_fourcc(*'MJPG'), 20.0, (2*args.out_size_w, args.out_size_h)) if os.path.exists(args.output_dir) is False: os.mkdir(args.output_dir) self.output_img_list = [] self.save_jpgs = args.save_jpgs def load_model(self): # 載入預訓練的天空摳圖模型 print('loading the best checkpoint...') checkpoint = torch.load(os.path.join(self.ckptdir, 'best_ckpt.pt'), map_location=device) self.net_G.load_state_dict(checkpoint['model_G_state_dict']) self.net_G.to(device) self.net_G.eval() def write_video(self, img_HD, syneth): frame = np.array(255.0 * syneth[:, :, ::-1], dtype=np.uint8) self.video_writer.write(frame) frame_cat = np.concatenate([img_HD, syneth], axis=1) frame_cat = np.array(255.0 * frame_cat[:, :, ::-1], dtype=np.uint8) self.video_writer_cat.write(frame_cat) # 定義結果緩衝區 self.output_img_list.append(frame_cat) def synthesize(self, img_HD, img_HD_prev): h, w, c = img_HD.shape img = cv2.resize(img_HD, (self.in_size_w, self.in_size_h)) img = np.array(img, dtype=np.float32) img = torch.tensor(img).permute([2, 0, 1]).unsqueeze(0) with torch.no_grad(): G_pred = self.net_G(img.to(device)) G_pred = torch.nn.functional.interpolate(G_pred, (h, w), mode='bicubic', align_corners=False) G_pred = G_pred[0, :].permute([1, 2, 0]) G_pred = torch.cat([G_pred, G_pred, G_pred], dim=-1) G_pred = np.array(G_pred.detach().cpu()) G_pred = np.clip(G_pred, a_max=1.0, a_min=0.0) skymask = self.skyboxengine.skymask_refinement(G_pred, img_HD) syneth = self.skyboxengine.skyblend(img_HD, img_HD_prev, skymask) return syneth, G_pred, skymask def cvtcolor_and_resize(self, img_HD): img_HD = cv2.cvtColor(img_HD, cv2.COLOR_BGR2RGB) img_HD = np.array(img_HD / 255., dtype=np.float32) img_HD = cv2.resize(img_HD, (self.out_size_w, self.out_size_h)) return img_HD def process_video(self): # 逐幀處理視訊 cap = cv2.VideoCapture(self.datadir) m_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) img_HD_prev = None for idx in range(m_frames): ret, frame = cap.read() if ret: img_HD = self.cvtcolor_and_resize(frame) if img_HD_prev is None: img_HD_prev = img_HD syneth, G_pred, skymask = self.synthesize(img_HD, img_HD_prev) self.write_video(img_HD, syneth) img_HD_prev = img_HD if (idx + 1) % 50 == 0: print(f'processing video, frame {idx + 1} / {m_frames} ... ') else: # 如果到達最後一幀 break
5、替換天空
替換後輸出的視訊為out.avi，前後對比的視訊為compare.avi
sf = SkyFilter(args) sf.process_video() initialize skybox... initialize network with normal loading the best checkpoint... processing video, frame 50 / 360 ... processing video, frame 100 / 360 ... no good point matched processing video, frame 150 / 360 ... processing video, frame 200 / 360 ... processing video, frame 250 / 360 ... processing video, frame 300 / 360 ... processing video, frame 350 / 360 ...
6、對比原視訊和替換後的視訊
video_name = "compare.avi" def arrayShow(img): _,ret = cv2.imencode('.jpg', img) return Image(data=ret) # 開啟一個視訊流 cap = cv2.VideoCapture(video_name) frame_id = 0 while True: try: clear_output(wait=True) # 清除之前的顯示 ret, frame = cap.read() # 讀取一幀圖片 if ret: frame_id += 1 cv2.putText(frame, str(frame_id), (5, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 1) # 畫frame_id tmp = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # 轉換色彩模式 img = arrayShow(frame) display(img) # 顯示圖片 time.sleep(0.05) # 執行緒睡眠一段時間再處理下一幀圖片 else: break except KeyboardInterrupt: cap.release() cap.release()
如果要生成自己的視訊，只要將test_videos中的sky.mp4視訊和skybox中的sky.jpg圖片替換成自己的視訊和圖片，然後重新一鍵執行就可以了。趕快來試一試吧，讓你的國慶大片更出彩！
華為雲社群祝大家國慶節快樂，度過一個開心的假期！
附錄
本案例源自華為雲AI Gallery：魔幻黑科技，可換天造物，秒變科幻大片！