update submodules

This reverts commit 25a58123a169dec4282b92bc1b24411278d0b65f.

Makefile

@@ -129,6 +129,7 @@ install-internal: downloads
 
 install: install-internal
 	mkdir -p $(STEAM_DIR)/compatibilitytools.d/
+	rm -rf $(STEAM_DIR)/compatibilitytools.d/$(_build_name)/files/ #remove proton's internal files, but preserve user_settings etc from top-level
 	cp -Rf --no-dereference --preserve=mode,links $(BUILD_ROOT)/compatibilitytools.d/$(_build_name) $(STEAM_DIR)/compatibilitytools.d/
 	echo "Proton installed to your local Steam installation"
 
@@ -163,8 +164,8 @@ dxvk: | $(BUILD_ROOT)/dxvk/lib/wine/dxvk
 dxvk: | $(BUILD_ROOT)/dxvk/lib64/wine/dxvk
 dxvk: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) dxvk && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/dxvk/*.dll $(BUILD_ROOT)/dxvk/lib/wine/dxvk/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/dxvk/*.dll $(BUILD_ROOT)/dxvk/lib64/wine/dxvk/
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/dxvk/*.dll $(BUILD_ROOT)/dxvk/lib/wine/dxvk/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/dxvk/*.dll $(BUILD_ROOT)/dxvk/lib64/wine/dxvk/
 
 dxvk-nvapi: | $(BUILD_ROOT)/dxvk-nvapi/lib/wine/nvapi
 dxvk-nvapi: | $(BUILD_ROOT)/dxvk-nvapi/lib64/wine/nvapi
@@ -177,8 +178,8 @@ vkd3d-proton: | $(BUILD_ROOT)/vkd3d-proton/lib/wine/vkd3d-proton
 vkd3d-proton: | $(BUILD_ROOT)/vkd3d-proton/lib64/wine/vkd3d-proton
 vkd3d-proton: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) vkd3d-proton && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/vkd3d-proton/*.dll $(BUILD_ROOT)/vkd3d-proton/lib/wine/vkd3d-proton/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/vkd3d-proton/*.dll $(BUILD_ROOT)/vkd3d-proton/lib64/wine/vkd3d-proton/
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/vkd3d-proton/*.dll $(BUILD_ROOT)/vkd3d-proton/lib/wine/vkd3d-proton/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/vkd3d-proton/*.dll $(BUILD_ROOT)/vkd3d-proton/lib64/wine/vkd3d-proton/
 
 lsteamclient: | $(BUILD_ROOT)/lsteamclient/lib/wine/i386-windows
 lsteamclient: | $(BUILD_ROOT)/lsteamclient/lib/wine/i386-unix
@@ -186,10 +187,10 @@ lsteamclient: | $(BUILD_ROOT)/lsteamclient/lib64/wine/x86_64-windows
 lsteamclient: | $(BUILD_ROOT)/lsteamclient/lib64/wine/x86_64-unix
 lsteamclient: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) lsteamclient && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/i386-windows/lsteamclient.dll $(BUILD_ROOT)/lsteamclient/lib/wine/i386-windows/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/i386-unix/lsteamclient.dll.so $(BUILD_ROOT)/lsteamclient/lib/wine/i386-unix/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/x86_64-windows/lsteamclient.dll $(BUILD_ROOT)/lsteamclient/lib64/wine/x86_64-windows/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/x86_64-unix/lsteamclient.dll.so $(BUILD_ROOT)/lsteamclient/lib64/wine/x86_64-unix/
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/i386-windows/lsteamclient.dll $(BUILD_ROOT)/lsteamclient/lib/wine/i386-windows/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/i386-unix/lsteamclient.dll.so $(BUILD_ROOT)/lsteamclient/lib/wine/i386-unix/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/x86_64-windows/lsteamclient.dll $(BUILD_ROOT)/lsteamclient/lib64/wine/x86_64-windows/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/x86_64-unix/lsteamclient.dll.so $(BUILD_ROOT)/lsteamclient/lib64/wine/x86_64-unix/
 
 vrclient: | $(BUILD_ROOT)/vrclient/lib/wine/i386-windows
 vrclient: | $(BUILD_ROOT)/vrclient/lib/wine/i386-unix
@@ -197,17 +198,17 @@ vrclient: | $(BUILD_ROOT)/vrclient/lib64/wine/x86_64-windows
 vrclient: | $(BUILD_ROOT)/vrclient/lib64/wine/x86_64-unix
 vrclient: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) vrclient && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/i386-windows/vrclient.dll $(BUILD_ROOT)/vrclient/lib/wine/i386-windows/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib/wine/i386-unix/vrclient.dll.so $(BUILD_ROOT)/vrclient/lib/wine/i386-unix/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/x86_64-windows/vrclient_x64.dll $(BUILD_ROOT)/vrclient/lib64/wine/x86_64-windows/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/x86_64-unix/vrclient_x64.dll.so $(BUILD_ROOT)/vrclient/lib64/wine/x86_64-unix/
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/i386-windows/vrclient.dll $(BUILD_ROOT)/vrclient/lib/wine/i386-windows/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib/wine/i386-unix/vrclient.dll.so $(BUILD_ROOT)/vrclient/lib/wine/i386-unix/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/x86_64-windows/vrclient_x64.dll $(BUILD_ROOT)/vrclient/lib64/wine/x86_64-windows/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/x86_64-unix/vrclient_x64.dll.so $(BUILD_ROOT)/vrclient/lib64/wine/x86_64-unix/
 
 wineopenxr: | $(BUILD_ROOT)/wineopenxr/lib64/wine/x86_64-windows
 wineopenxr: | $(BUILD_ROOT)/wineopenxr/lib64/wine/x86_64-unix
 wineopenxr: downloads
 	$(MAKE) $(MFLAGS) $(MAKEOVERRIDES) -C $(BUILD_DIR)/ $(UNSTRIPPED) wineopenxr && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/x86_64-windows/wineopenxr.dll $(BUILD_ROOT)/wineopenxr/lib64/wine/x86_64-windows/ && \
-	cp -f $(BUILD_DIR)/dist/dist/lib64/wine/x86_64-unix/wineopenxr.dll.so $(BUILD_ROOT)/wineopenxr/lib64/wine/x86_64-unix/
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/x86_64-windows/wineopenxr.dll $(BUILD_ROOT)/wineopenxr/lib64/wine/x86_64-windows/ && \
+	cp -f $(BUILD_DIR)/dist/files/lib64/wine/x86_64-unix/wineopenxr.dll.so $(BUILD_ROOT)/wineopenxr/lib64/wine/x86_64-unix/
 
 battleye: | $(BUILD_ROOT)/battleye/v1/lib/wine/i386-windows
 battleye: | $(BUILD_ROOT)/battleye/v1/lib/wine/i386-unix

Makefile.in

@@ -143,7 +143,7 @@ endif
 ##
 
 DST_BASE := $(OBJ)/dist
-DST_DIR := $(DST_BASE)/dist
+DST_DIR := $(DST_BASE)/files
 DST_LIBDIR32 := $(DST_DIR)/lib
 DST_LIBDIR64 := $(DST_DIR)/lib64
 DEPLOY_DIR := ./deploy
@@ -186,12 +186,13 @@ COMPAT_MANIFEST_TEMPLATE := $(SRCDIR)/compatibilitytool.vdf.template
 LICENSE := $(SRCDIR)/dist.LICENSE
 OFL_LICENSE := $(SRCDIR)/fonts/liberation-fonts/LICENSE
 AV1_PATENTS := $(SRCDIR)/dav1d/doc/PATENTS
+STEAMPIPE_FIXUPS_PY := $(SRCDIR)/steampipe_fixups.py
 
 GECKO_VER := 2.47.2
 GECKO32_TARBALL := wine-gecko-$(GECKO_VER)-x86.tar.xz
 GECKO64_TARBALL := wine-gecko-$(GECKO_VER)-x86_64.tar.xz
 
-WINEMONO_VER := 7.1.2
+WINEMONO_VER := 7.1.3
 WINEMONO_TARBALL := wine-mono-$(WINEMONO_VER)-x86.tar.xz
 
 FONTS := $(SRCDIR)/fonts
@@ -274,9 +275,10 @@ DIST_TARGETS := $(DIST_COPY_TARGETS) $(DIST_OVR32) $(DIST_OVR64) \
                 $(DIST_COMPAT_MANIFEST) $(DIST_LICENSE) $(DIST_TOOLMANIFEST) \
                 $(DIST_OFL_LICENSE) $(DIST_AV1_PATENTS) $(DIST_FONTS)
 
-DEPLOY_COPY_TARGETS := $(DIST_COPY_TARGETS) $(DIST_VERSION) $(DIST_LICENSE) \
-                       $(DIST_TOOLMANIFEST) $(DIST_OFL_LICENSE) $(DIST_AV1_PATENTS)
-REDIST_COPY_TARGETS := $(DEPLOY_COPY_TARGETS) $(DIST_COMPAT_MANIFEST)
+BASE_COPY_TARGETS := $(DIST_COPY_TARGETS) $(DIST_VERSION) $(DIST_LICENSE) \
+                     $(DIST_TOOLMANIFEST) $(DIST_OFL_LICENSE) $(DIST_AV1_PATENTS) $(DST_DIR)
+DEPLOY_COPY_TARGETS := $(BASE_COPY_TARGETS) $(STEAMPIPE_FIXUPS_PY)
+REDIST_COPY_TARGETS := $(BASE_COPY_TARGETS) $(DIST_COMPAT_MANIFEST)
 
 $(DIST_LICENSE): $(LICENSE)
 	cp -a $< $@
@@ -371,24 +373,21 @@ dist: $(DIST_TARGETS) all-dist dist_wineopenxr | $(DST_DIR)
 	echo `date '+%s'` `GIT_DIR=$(abspath $(SRCDIR)/.git) git describe --tags` > $(DIST_VERSION)
 
 deploy: dist | $(filter-out dist deploy install redist,$(MAKECMDGOALS))
-	mkdir -p $(DEPLOY_DIR) && \
-	cp -a $(DEPLOY_COPY_TARGETS) $(DEPLOY_DIR) && \
-	tar -C $(DST_DIR) -c . > $(DEPLOY_DIR)/proton_dist.tar
-	@echo "Created deployment archive at "$(DEPLOY_DIR)"/proton_dist.tar"
+	mkdir -p $(DEPLOY_DIR)
+	cp -af --no-dereference --preserve=mode,links $(DEPLOY_COPY_TARGETS) $(DEPLOY_DIR)
+	python3 $(STEAMPIPE_FIXUPS_PY) process $(DEPLOY_DIR)
 
 install: dist | $(filter-out dist deploy install redist,$(MAKECMDGOALS))
 	if [ ! -d $(STEAM_DIR) ]; then echo >&2 "!! "$(STEAM_DIR)" does not exist, cannot install"; return 1; fi
 	mkdir -p $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
+	# Use -r instead of -a for sshfs
 	cp -rf --no-dereference --preserve=mode,links $(DST_BASE)/* $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
-	cp -f $(DIST_VERSION) $(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)/dist/
 	@echo "Installed Proton to "$(STEAM_DIR)/compatibilitytools.d/$(BUILD_NAME)
 	@echo "You may need to restart Steam to select this tool"
 
 redist: dist | $(filter-out dist deploy install redist,$(MAKECMDGOALS))
 	mkdir -p $(REDIST_DIR)
-	cp -a $(REDIST_COPY_TARGETS) $(REDIST_DIR)
-	tar -C $(DST_DIR) -c . | gzip -c -1 > $(REDIST_DIR)/proton_dist.tar.gz
-	@echo "Created redistribution tarball at "$(REDIST_DIR)"/proton_dist.tar.gz"
+	cp -af --no-dereference --preserve=mode,links $(REDIST_COPY_TARGETS) $(REDIST_DIR)
 
 .PHONY: module32 module64 module
 

default_pfx.py

@@ -71,6 +71,14 @@ def setup_dll_symlinks(default_pfx_dir, dist_dir):
                 os.unlink(filename)
                 make_relative_symlink(target, filename)
 
+#steampipe can't handle filenames with colons, so we remove them here
+#and restore them in the proton script
+def fixup_drive_links(default_pfx_dir):
+    for walk_dir, dirs, files in os.walk(os.path.join(default_pfx_dir, "dosdevices")):
+        for dir_ in dirs:
+            if ":" in dir_:
+                os.remove(os.path.join(walk_dir, dir_))
+
 def make_default_pfx(default_pfx_dir, dist_dir, runtime):
     local_env = dict(os.environ)
 
@@ -94,6 +102,7 @@ def make_default_pfx(default_pfx_dir, dist_dir, runtime):
 
         env=local_env, check=True)
     setup_dll_symlinks(default_pfx_dir, dist_dir)
+    fixup_drive_links(default_pfx_dir)
 
 if __name__ == '__main__':
     import sys

docker/proton.Dockerfile.in

@@ -52,11 +52,13 @@ RUN bash -c 'mkdir -p /usr/lib/ccache && ls /usr/bin/{,*-}{cc,c++,gcc,g++}{,-[0-
 ENV PATH=/usr/lib/ccache:$PATH
 
 RUN apt-get install -y \
+  fontforge \
   libxpresent-dev \
   libxpresent-dev:i386 \
   python3-pefile \
   libcapstone-dev \
   libcapstone-dev:i386 \
+  libutfcpp-dev \
   yasm \
 && rm -rf /opt/usr/share/doc /opt/usr/share/info /opt/usr/share/man \
 && rm -rf /var/lib/apt/lists/*

dxvk

@@ -1 +1 @@
-Subproject commit 1216466be7917ea9336cf3172a4d119a4c9f97f2
+Subproject commit 0ade12dc83e7418bd4ebde440f50cbd950fc06a5

make/pefixup.py

@@ -1,25 +0,0 @@
-#!/usr/bin/env python3
-
-import sys
-import os
-import stat
-import pefile
-
-for path in sys.argv[1:]:
-    pe = pefile.PE(path)
-
-    for section in pe.sections:
-        if section.Name.decode("utf-8")[0:5] == ".text":
-            section.Characteristics &= ~pefile.SECTION_CHARACTERISTICS['IMAGE_SCN_CNT_INITIALIZED_DATA']
-        section.Characteristics &= ~pefile.SECTION_CHARACTERISTICS['IMAGE_SCN_ALIGN_MASK']
-
-    pe.OPTIONAL_HEADER.CheckSum = pe.generate_checksum()
-
-    perm = stat.S_IMODE(os.stat(path).st_mode)
-    if (perm & stat.S_IWUSR) == 0:
-        os.chmod(path, perm | stat.S_IWUSR)
-
-    pe.write(path)
-
-    if (perm & stat.S_IWUSR) == 0:
-        os.chmod(path, perm)

make/rules-common.mk

@@ -60,13 +60,10 @@ $$(OBJ)/.$(1)-dist$(3):
 	cd $$($(2)_LIBDIR$(3)) && find -type l -printf '%p\0$$(DST_LIBDIR$(3))/%p\0' | xargs $(--verbose?) -0 -r -P$$(J) -n2 cp -a
 	cd $$($(2)_LIBDIR$(3)) && find -type f -not '(' -iname '*.pc' -or -iname '*.cmake' -or -iname '*.a' -or -iname '*.la' -or -iname '*.def' ')' \
 	    -printf '--only-keep-debug\0%p\0$$(DST_LIBDIR$(3))/%p.debug\0' | \
-	    xargs $(--verbose?) -0 -r -P$$(J) -n3 objcopy --file-alignment=4096
+	    xargs $(--verbose?) -0 -r -P$$(J) -n3 objcopy --file-alignment=4096 --set-section-flags .text=contents,alloc,load,readonly,code
 	cd $$($(2)_LIBDIR$(3)) && find -type f -not '(' -iname '*.pc' -or -iname '*.cmake' -or -iname '*.a' -or -iname '*.la' -or -iname '*.def' ')' \
 	    -printf '--add-gnu-debuglink=$$(DST_LIBDIR$(3))/%p.debug\0--strip-debug\0%p\0$$(DST_LIBDIR$(3))/%p\0' | \
-	    xargs $(--verbose?) -0 -r -P$$(J) -n4 objcopy --file-alignment=4096
-	cd $$($(2)_LIBDIR$(3)) && find -type f \( -name '*.dll' -o -name '*.drv' \) \
-	    -printf '$$(DST_LIBDIR$(3))/%p\0' | \
-	    xargs $(--verbose?) -0 -r -P$$(J) -n1 $$(SRC)/make/pefixup.py
+	    xargs $(--verbose?) -0 -r -P$$(J) -n4 objcopy --file-alignment=4096 --set-section-flags .text=contents,alloc,load,readonly,code
 	touch $$@
 else
 $$(OBJ)/.$(1)-dist$(3):
@@ -78,10 +75,7 @@ $$(OBJ)/.$(1)-dist$(3):
 	    -printf '$$(DST_LIBDIR$(3))/%p.debug\0' | xargs $(--verbose?) -0 -r -P$$(J) rm -f
 	cd $$($(2)_LIBDIR$(3)) && find -type f -not '(' -iname '*.pc' -or -iname '*.cmake' -or -iname '*.a' -or -iname '*.la' -or -iname '*.def' ')' \
 	    -printf '--strip-debug\0%p\0$$(DST_LIBDIR$(3))/%p\0' | \
-	    xargs $(--verbose?) -0 -r -P$$(J) -n3 objcopy --file-alignment=4096
-	cd $$($(2)_LIBDIR$(3)) && find -type f \( -name '*.dll' -o -name '*.drv' \) \
-	    -printf '$$(DST_LIBDIR$(3))/%p\0' | \
-	    xargs $(--verbose?) -0 -r -P$$(J) -n1 $$(SRC)/make/pefixup.py
+	    xargs $(--verbose?) -0 -r -P$$(J) -n3 objcopy --file-alignment=4096 --set-section-flags .text=contents,alloc,load,readonly,code
 	touch $$@
 endif
 endif

proton

@@ -36,6 +36,13 @@ CURRENT_PREFIX_VERSION="7.0-100"
 PFX="Proton: "
 ld_path_var = "LD_LIBRARY_PATH"
 
+def file_exists(s, *, follow_symlinks):
+    if follow_symlinks:
+        #'exists' returns False on broken symlinks
+        return os.path.exists(s)
+    #'lexists' returns True on broken symlinks
+    return os.path.lexists(s)
+
 def nonzero(s):
     return len(s) > 0 and s != "0"
 
@@ -70,7 +77,7 @@ def file_is_wine_builtin_dll(path):
                 )):
             # This may be a broken link to a dll in a removed Proton install
             return True
-    if not os.path.exists(path):
+    if not file_exists(path, follow_symlinks=True):
         return False
     try:
         sfile = open(path, "rb")
@@ -82,6 +89,9 @@ def file_is_wine_builtin_dll(path):
 
 def makedirs(path):
     try:
+        #replace broken symlinks with a new directory
+        if os.path.islink(path) and not file_exists(path, follow_symlinks=True):
+            os.remove(path)
         os.makedirs(path)
     except OSError:
         #already exists
@@ -103,17 +113,17 @@ def merge_user_dir(src, dst):
 
         #we only want to copy into directories which don't already exist. games
         #may not react well to two save directory instances being merged.
-        if not os.path.exists(dst_dir) or os.path.samefile(dst_dir, dst):
+        if not file_exists(dst_dir, follow_symlinks=True) or os.path.samefile(dst_dir, dst):
             makedirs(dst_dir)
             for dir_ in dirs:
                 src_file = os.path.join(src_dir, dir_)
                 dst_file = os.path.join(dst_dir, dir_)
-                if os.path.islink(src_file) and not os.path.exists(dst_file):
+                if os.path.islink(src_file) and not file_exists(dst_file, follow_symlinks=True):
                     try_copy(src_file, dst_file, copy_metadata=True, follow_symlinks=False)
             for file_ in files:
                 src_file = os.path.join(src_dir, file_)
                 dst_file = os.path.join(dst_dir, file_)
-                if not os.path.exists(dst_file):
+                if not file_exists(dst_file, follow_symlinks=True):
                     try_copy(src_file, dst_file, copy_metadata=True, follow_symlinks=False)
         else:
             extant_dirs += dst_dir
@@ -127,7 +137,7 @@ def try_copy(src, dst, prefix=None, add_write_perm=True, copy_metadata=False, op
         if os.path.isdir(dst):
             dst = os.path.join(dst, os.path.basename(src))
 
-        if os.path.lexists(dst):
+        if file_exists(dst, follow_symlinks=False):
             os.remove(dst)
         elif track_file and prefix is not None:
             track_file.write(os.path.relpath(dst, prefix) + '\n')
@@ -141,10 +151,10 @@ def try_copy(src, dst, prefix=None, add_write_perm=True, copy_metadata=False, op
             new_mode = os.lstat(dst).st_mode | stat.S_IWUSR | stat.S_IWGRP
             os.chmod(dst, new_mode)
 
-        if not os.path.exists(src + '.debug'):
+        if not file_exists(src + '.debug', follow_symlinks=True):
             link_debug = False
 
-        if os.path.lexists(dst + '.debug'):
+        if file_exists(dst + '.debug', follow_symlinks=False):
             os.remove(dst + '.debug')
         elif link_debug:
             track_file.write(os.path.relpath(dst + '.debug', prefix) + '\n')
@@ -169,9 +179,9 @@ def try_copyfile(src, dst):
     try:
         if os.path.isdir(dst):
             dstfile = dst + "/" + os.path.basename(src)
-            if os.path.lexists(dstfile):
+            if file_exists(dstfile, follow_symlinks=False):
                 os.remove(dstfile)
-        elif os.path.lexists(dst):
+        elif file_exists(dst, follow_symlinks=False):
             os.remove(dst)
         shutil.copyfile(src, dst)
     except PermissionError as e:
@@ -206,17 +216,17 @@ def setup_game_dir_drive():
         if "gamedrive" in g_session.compat_config:
             library_dir = try_get_game_library_dir()
             if not library_dir:
-                if os.path.lexists(gamedrive_path):
+                if file_exists(gamedrive_path, follow_symlinks=False):
                     os.remove(gamedrive_path)
             else:
-                if os.path.lexists(gamedrive_path):
+                if file_exists(gamedrive_path, follow_symlinks=False):
                     cur_tgt = os.readlink(gamedrive_path)
                     if cur_tgt != library_dir:
                         os.remove(gamedrive_path)
                         os.symlink(library_dir, gamedrive_path)
                 else:
                     os.symlink(library_dir, gamedrive_path)
-        elif os.path.lexists(gamedrive_path):
+        elif file_exists(gamedrive_path, follow_symlinks=False):
             os.remove(gamedrive_path)
 
 # Function to find the installed location of DLL files for use by Wine/Proton
@@ -295,7 +305,7 @@ def find_nvidia_wine_dll_dir():
     nvidia_wine_dir = os.path.join(os.path.dirname(libglx_nvidia_realpath), "nvidia", "wine")
 
     # Check that nvngx.dll exists here, or fail
-    if os.path.exists(os.path.join(nvidia_wine_dir, "nvngx.dll")):
+    if file_exists(os.path.join(nvidia_wine_dir, "nvngx.dll"), follow_symlinks=True):
         return nvidia_wine_dir
 
     return None
@@ -322,15 +332,15 @@ def set_dir_casefold_bit(dir_path):
 class Proton:
     def __init__(self, base_dir):
         self.base_dir = base_dir + "/"
-        self.dist_dir = self.path("dist/")
-        self.bin_dir = self.path("dist/bin/")
-        self.lib_dir = self.path("dist/lib/")
-        self.lib64_dir = self.path("dist/lib64/")
-        self.fonts_dir = self.path("dist/share/fonts/")
-        self.wine_fonts_dir = self.path("dist/share/wine/fonts/")
-        self.wine_inf = self.path("dist/share/wine/wine.inf")
+        self.dist_dir = self.path("files/")
+        self.bin_dir = self.path("files/bin/")
+        self.lib_dir = self.path("files/lib/")
+        self.lib64_dir = self.path("files/lib64/")
+        self.fonts_dir = self.path("files/share/fonts/")
+        self.wine_fonts_dir = self.path("files/share/wine/fonts/")
+        self.wine_inf = self.path("files/share/wine/wine.inf")
         self.version_file = self.path("version")
-        self.default_pfx_dir = self.path("dist/share/default_pfx/")
+        self.default_pfx_dir = self.path("files/share/default_pfx/")
         self.user_settings_file = self.path("user_settings.py")
         self.wine_bin = self.bin_dir + "wine"
         self.wine64_bin = self.bin_dir + "wine64"
@@ -340,28 +350,34 @@ class Proton:
     def path(self, d):
         return self.base_dir + d
 
-    def need_tarball_extraction(self):
-        '''Checks if the proton_dist tarball archive must be extracted. Returns true if extraction is needed, false otherwise'''
-        return not os.path.exists(self.dist_dir) or \
-            not os.path.exists(self.path("dist/version")) or \
-            not filecmp.cmp(self.version_file, self.path("dist/version"))
+    def cleanup_legacy_dist(self):
+        old_dist_dir = self.path("dist/")
+        if file_exists(old_dist_dir, follow_symlinks=True):
+            with self.dist_lock:
+                if file_exists(old_dist_dir, follow_symlinks=True):
+                    shutil.rmtree(old_dist_dir)
 
-    def extract_tarball(self):
-        with self.dist_lock:
-            if self.need_tarball_extraction():
-                if os.path.exists(self.dist_dir):
-                    shutil.rmtree(self.dist_dir)
-                tar = None
-                for sf in ["", ".xz", ".bz2", ".gz"]:
-                    if os.path.exists(self.path("proton_dist.tar" + sf)):
-                        tar = tarfile.open(self.path("proton_dist.tar" + sf), mode="r:*")
-                        break
-                if not tar:
-                    log("No proton_dist tarball??")
-                    sys.exit(1)
-                tar.extractall(path=self.dist_dir)
-                tar.close()
-                try_copy(self.version_file, self.dist_dir)
+    def do_steampipe_fixups(self):
+        fixups_json = self.path("steampipe_fixups.json")
+        fixups_mtime = self.path("files/steampipe_fixups_mtime")
+
+        if file_exists(fixups_json, follow_symlinks=True):
+            with self.dist_lock:
+                import steampipe_fixups
+
+                current_fixup_mtime = None
+                if file_exists(fixups_mtime, follow_symlinks=True):
+                    with open(fixups_mtime, "r") as f:
+                        current_fixup_mtime = f.readline().strip()
+
+                new_fixup_mtime = getmtimestr(fixups_json)
+
+                if current_fixup_mtime != new_fixup_mtime:
+                    result_code = steampipe_fixups.do_restore(self.base_dir, fixups_json)
+
+                    if result_code == 0:
+                        with open(fixups_mtime, "w") as f:
+                            f.write(new_fixup_mtime + "\n")
 
     def missing_default_prefix(self):
         '''Check if the default prefix dir is missing. Returns true if missing, false if present'''
@@ -390,7 +406,7 @@ class CompatData:
         return self.base_dir + d
 
     def remove_tracked_files(self):
-        if not os.path.exists(self.tracked_files_file):
+        if not file_exists(self.tracked_files_file, follow_symlinks=True):
             log("Prefix has no tracked_files??")
             return
 
@@ -398,7 +414,7 @@ class CompatData:
             dirs = []
             for f in tracked_files:
                 path = self.prefix_dir + f.strip()
-                if os.path.exists(path):
+                if file_exists(path, follow_symlinks=False):
                     if os.path.isfile(path) or os.path.islink(path):
                         os.remove(path)
                     else:
@@ -438,21 +454,21 @@ class CompatData:
                     (int(new_proton_maj) == int(old_proton_maj) and \
                      int(new_proton_min) < int(old_proton_min)):
                 log("Removing newer prefix")
-                if old_proton_ver == "3.7" and not os.path.exists(self.tracked_files_file):
+                if old_proton_ver == "3.7" and not file_exists(self.tracked_files_file, follow_symlinks=True):
                     #proton 3.7 did not generate tracked_files, so copy it into place first
                     try_copy(g_proton.path("proton_3.7_tracked_files"), self.tracked_files_file)
                 self.remove_tracked_files()
                 return
 
             if old_proton_ver == "3.7" and old_prefix_ver == "1":
-                if not os.path.exists(self.prefix_dir + "/drive_c/windows/syswow64/kernel32.dll"):
+                if not file_exists(self.prefix_dir + "/drive_c/windows/syswow64/kernel32.dll", follow_symlinks=True):
                     #shipped a busted 64-bit-only installation on 20180822. detect and wipe clean
                     log("Detected broken 64-bit-only installation, re-creating prefix.")
                     shutil.rmtree(self.prefix_dir)
                     return
 
             #replace broken .NET installations with wine-mono support
-            if os.path.exists(self.prefix_dir + "/drive_c/windows/Microsoft.NET/NETFXRepair.exe") and \
+            if file_exists(self.prefix_dir + "/drive_c/windows/Microsoft.NET/NETFXRepair.exe", follow_symlinks=True) and \
                     file_is_wine_builtin_dll(self.prefix_dir + "/drive_c/windows/system32/mscoree.dll"):
                 log("Broken .NET installation detected, switching to wine-mono.")
                 #deleting this directory allows wine-mono to work
@@ -533,7 +549,7 @@ class CompatData:
             stale_builtins = [self.prefix_dir + "/drive_c/windows/system32/amd_ags_x64.dll",
                               self.prefix_dir + "/drive_c/windows/syswow64/amd_ags_x64.dll" ]
             for builtin in stale_builtins:
-                if os.path.lexists(builtin) and file_is_wine_builtin_dll(builtin):
+                if file_exists(builtin, follow_symlinks=False) and file_is_wine_builtin_dll(builtin):
                     log("Removing stale builtin " + builtin)
                     os.remove(builtin)
 
@@ -563,18 +579,18 @@ class CompatData:
                 if len(rel_dir) > 0:
                     rel_dir = rel_dir + "/"
                 dst_dir = src_dir.replace(g_proton.default_pfx_dir, self.prefix_dir, 1)
-                if not os.path.lexists(dst_dir):
-                    os.makedirs(dst_dir)
+                if not file_exists(dst_dir, follow_symlinks=True):
+                    makedirs(dst_dir)
                     tracked_files.write(rel_dir + "\n")
                 for dir_ in dirs:
                     src_file = os.path.join(src_dir, dir_)
                     dst_file = os.path.join(dst_dir, dir_)
-                    if os.path.islink(src_file) and not os.path.exists(dst_file):
+                    if os.path.islink(src_file) and not file_exists(dst_file, follow_symlinks=True):
                         self.pfx_copy(src_file, dst_file)
                 for file_ in files:
                     src_file = os.path.join(src_dir, file_)
                     dst_file = os.path.join(dst_dir, file_)
-                    if not os.path.exists(dst_file):
+                    if not file_exists(dst_file, follow_symlinks=True):
                         self.pfx_copy(src_file, dst_file)
                         tracked_files.write(rel_dir + file_ + "\n")
         # Set .update-timestamp so Wine doesn't try to update the prefix.
@@ -595,8 +611,8 @@ class CompatData:
                 if len(rel_dir) > 0:
                     rel_dir = rel_dir + "/"
                 dst_dir = src_dir.replace(g_proton.default_pfx_dir, self.prefix_dir, 1)
-                if not os.path.lexists(dst_dir):
-                    os.makedirs(dst_dir)
+                if not file_exists(dst_dir, follow_symlinks=True):
+                    makedirs(dst_dir)
                     tracked_files.write(rel_dir + "\n")
                 for file_ in files:
                     src_file = os.path.join(src_dir, file_)
@@ -606,7 +622,7 @@ class CompatData:
                         continue
                     if file_is_wine_builtin_dll(dst_file):
                         os.unlink(dst_file)
-                    elif os.path.lexists(dst_file):
+                    elif file_exists(dst_file, follow_symlinks=False):
                         # builtin library was replaced
                         continue
                     else:
@@ -639,7 +655,7 @@ class CompatData:
         for p in fontsmap:
             lname = os.path.join(windowsfonts, p[2])
             fname = os.path.join(p[0], p[1])
-            if os.path.lexists(lname):
+            if file_exists(lname, follow_symlinks=False):
                 if os.path.islink(lname):
                     os.remove(lname)
                     os.symlink(fname, lname)
@@ -664,15 +680,15 @@ class CompatData:
 
             #running unofficial Proton/Wine builds against a Proton prefix could
             #create an infinite symlink loop. detect this and clean it up.
-            if os.path.lexists(new) and os.path.islink(new) and os.readlink(new).endswith(old):
+            if file_exists(new, follow_symlinks=False) and os.path.islink(new) and os.readlink(new).endswith(old):
                 os.remove(new)
 
             old = self.prefix_dir + old
 
-            if os.path.lexists(old) and not os.path.islink(old):
+            if file_exists(old, follow_symlinks=False) and not os.path.islink(old):
                 merge_user_dir(src=old, dst=new)
                 os.rename(old, old + " BACKUP")
-            if not os.path.lexists(old):
+            if not file_exists(old, follow_symlinks=False):
                 makedirs(os.path.dirname(old))
                 os.symlink(src=link, dst=old)
             elif os.path.islink(old) and not (os.readlink(old) == link):
@@ -687,7 +703,7 @@ class CompatData:
         config_dir = self.prefix_dir + "drive_c/users/steamuser/My Documents/My Games/FINAL FANTASY XIV - A Realm Reborn/"
         config_file = config_dir + "FFXIV_BOOT.cfg"
 
-        if os.path.exists(config_file):
+        if file_exists(config_file, follow_symlinks=True):
             with open(config_file, "r") as file:
                 config_content = file.read()
             config_content = config_content.replace("Browser\t2", "Browser\t1")
@@ -701,7 +717,7 @@ class CompatData:
 
     def setup_prefix(self):
         with self.prefix_lock:
-            if os.path.exists(self.version_file):
+            if file_exists(self.version_file, follow_symlinks=True):
                 with open(self.version_file, "r") as f:
                     old_ver = f.readline().strip()
             else:
@@ -709,15 +725,21 @@ class CompatData:
 
             self.upgrade_pfx(old_ver)
 
-            if not os.path.exists(self.prefix_dir):
+            if not file_exists(self.prefix_dir, follow_symlinks=True):
                 makedirs(self.prefix_dir + "/drive_c")
                 set_dir_casefold_bit(self.prefix_dir + "/drive_c")
 
-            if not os.path.exists(self.prefix_dir + "/user.reg"):
+            if not file_exists(self.prefix_dir + "/user.reg", follow_symlinks=True):
                 self.copy_pfx()
 
             self.migrate_user_paths()
 
+            if not file_exists(self.prefix_dir + "/dosdevices/c:", follow_symlinks=False):
+                os.symlink("../drive_c", self.prefix_dir + "/dosdevices/c:")
+
+            if not file_exists(self.prefix_dir + "/dosdevices/z:", follow_symlinks=False):
+                os.symlink("/", self.prefix_dir + "/dosdevices/z:")
+
             # collect configuration info
             steamdir = os.environ["STEAM_COMPAT_CLIENT_INSTALL_PATH"]
 
@@ -876,13 +898,13 @@ class CompatData:
                 else:
                     nvapi64_dll = self.prefix_dir + "drive_c/windows/system32/nvapi64.dll"
                     nvapi32_dll = self.prefix_dir + "drive_c/windows/syswow64/nvapi.dll"
-                    if os.path.exists(nvapi64_dll):
+                    if file_exists(nvapi64_dll, follow_symlinks=False):
                         os.unlink(nvapi64_dll)
-                    if os.path.exists(nvapi64_dll + '.debug'):
+                    if file_exists(nvapi64_dll + '.debug', follow_symlinks=False):
                         os.unlink(nvapi64_dll + '.debug')
-                    if os.path.exists(nvapi32_dll):
+                    if file_exists(nvapi32_dll, follow_symlinks=False):
                         os.unlink(nvapi32_dll)
-                    if os.path.exists(nvapi32_dll + '.debug'):
+                    if file_exists(nvapi32_dll + '.debug', follow_symlinks=False):
                         os.unlink(nvapi32_dll + '.debug')
 
                 # Try to detect known DLLs that ship with the NVIDIA Linux Driver
@@ -911,19 +933,6 @@ def comma_escaped(s):
         idx = idx - 1
     return escaped
 
-#hopefully short-lived, app-specific workarounds for Proton bugs
-def default_compat_config():
-    ret = set()
-    if "SteamAppId" in os.environ:
-        appid = os.environ["SteamAppId"]
-        if appid == "1172470": #Apex Legends
-            ret.add("noesync")
-            ret.add("nofsync")
-        elif appid == "105800": #PixelJunk Eden
-            ret.add("noesync")
-            ret.add("nofsync")
-    return ret
-
 class Session:
     def __init__(self):
         self.log_file = None
@@ -935,7 +944,7 @@ class Session:
                 "beclient_x64.dll": "b,n",
         }
 
-        self.compat_config = default_compat_config()
+        self.compat_config = set()
         self.cmdlineappend = []
 
         if "STEAM_COMPAT_CONFIG" in os.environ:
@@ -980,7 +989,7 @@ class Session:
 
         if "STEAM_COMPAT_MEDIA_PATH" in os.environ:
             old_audiofoz_path = os.environ["STEAM_COMPAT_MEDIA_PATH"] + "/audio.foz"
-            if os.path.lexists(old_audiofoz_path):
+            if file_exists(old_audiofoz_path, follow_symlinks=False):
                 os.remove(old_audiofoz_path)
             self.env["MEDIACONV_AUDIO_DUMP_FILE"] = os.environ["STEAM_COMPAT_MEDIA_PATH"] + "/audiov2.foz"
             self.env["MEDIACONV_VIDEO_DUMP_FILE"] = os.environ["STEAM_COMPAT_MEDIA_PATH"] + "/video.foz"
@@ -1025,7 +1034,7 @@ class Session:
 
             lfile_path = basedir + "/steam-" + os.environ["SteamGameId"] + ".log"
 
-            if os.path.exists(lfile_path):
+            if file_exists(lfile_path, follow_symlinks=False):
                 os.remove(lfile_path)
 
         makedirs(basedir)
@@ -1037,7 +1046,7 @@ class Session:
 
         #load environment overrides
         used_user_settings = {}
-        if os.path.exists(g_proton.user_settings_file):
+        if file_exists(g_proton.user_settings_file, follow_symlinks=True):
             try:
                 import user_settings
                 for key, value in user_settings.user_settings.items():
@@ -1054,12 +1063,14 @@ class Session:
             self.env.setdefault("WINEDEBUG", "+timestamp,+pid,+tid,+seh,+unwind,+debugstr,+loaddll,+mscoree")
             self.env.setdefault("DXVK_LOG_LEVEL", "info")
             self.env.setdefault("VKD3D_DEBUG", "warn")
+            self.env.setdefault("VKD3D_SHADER_DEBUG", "fixme")
             self.env.setdefault("WINE_MONO_TRACE", "E:System.NotImplementedException")
 
         #for performance, logging is disabled by default; override with user_settings.py
         self.env.setdefault("WINEDEBUG", "-all")
         self.env.setdefault("DXVK_LOG_LEVEL", "none")
         self.env.setdefault("VKD3D_DEBUG", "none")
+        self.env.setdefault("VKD3D_SHADER_DEBUG", "none")
 
         #disable XIM support until libx11 >= 1.7 is widespread
         self.env.setdefault("WINE_ALLOW_XIM", "0")
@@ -1350,8 +1361,8 @@ if __name__ == "__main__":
 
     g_proton = Proton(os.path.dirname(sys.argv[0]))
 
-    if g_proton.need_tarball_extraction():
-        g_proton.extract_tarball()
+    g_proton.cleanup_legacy_dist()
+    g_proton.do_steampipe_fixups()
 
     g_compatdata = CompatData(os.environ["STEAM_COMPAT_DATA_PATH"])
 

steampipe_fixups.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+
+#Steampipe doesn't support certain unix-y things which may be required by
+#native Linux applications. This file will process a directory of Linux files
+#and store the file properties into a manifest file. After a round trip through
+#Steampipe, the original file properties can be restored using this same
+#script.
+
+import json
+import os
+import secrets
+import stat
+
+DEFAULT_MANIFEST_NAME = "steampipe_fixups.json"
+
+def usage():
+    print("Usage:")
+    print("\t" + sys.argv[0] + "\tprepare\t<path to directory to process>\t[manifest output file]")
+    print("\t\tProcess the given path and output the manifest file to the given path, or <path/" + DEFAULT_MANIFEST_NAME + "> if unspecified.")
+    print("")
+    print("\t" + sys.argv[0] + "\trestore\t<path to directory to process>\t[manifest file]")
+    print("\t\tRestore the given path using the manifest file, or <path/" + DEFAULT_MANIFEST_NAME + "> if unspecified.")
+
+empty_dirs = []
+no_write_paths = []
+
+def canonicalize(path, prefix):
+    return path.replace(prefix, "", 1).lstrip('/')
+
+def process_dir(path):
+    for root, subdirs, files in os.walk(path):
+        if len(subdirs) == 0 and len(files) == 0:
+            empty_dirs.append(canonicalize(root, path))
+
+        for file_ in files:
+            this_file = os.path.join(root, file_)
+            stat_result = os.lstat(this_file)
+            if (stat_result.st_mode & stat.S_IWUSR) == 0:
+                no_write_paths.append(canonicalize(this_file, path))
+
+    return 0
+
+def write_manifest(manifest):
+    out = open(manifest, "w")
+    json.dump(
+        {
+            "id": str(secrets.randbits(32)), #we need steampipe to update this file for every build
+            "empty_dirs": empty_dirs,
+            "no_write_paths": no_write_paths,
+        },
+        out,
+        indent = 4,
+        sort_keys = True
+    )
+    return 0
+
+def do_process(path, manifest):
+    if os.path.exists(manifest):
+        os.remove(manifest)
+
+    ret = process_dir(path)
+    if ret != 0:
+        return ret
+
+    #output should be deterministic
+    empty_dirs.sort()
+    no_write_paths.sort()
+
+    ret = write_manifest(manifest)
+    if ret != 0:
+        return ret
+
+    return 0
+
+def do_restore(path, manifest):
+    loaded = json.load(open(manifest, "r"))
+
+    empty_dirs = loaded["empty_dirs"]
+    no_write_paths = loaded["no_write_paths"]
+
+    for empty_dir in empty_dirs:
+        try:
+            os.makedirs(os.path.join(path, empty_dir))
+        except OSError:
+            #already exists
+            pass
+
+    for file_ in no_write_paths:
+        this_file = os.path.join(path, file_)
+        stat_result = os.lstat(this_file)
+        os.chmod(this_file,
+                stat_result.st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
+
+    return 0
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) < 3 or len(sys.argv) > 4:
+        usage()
+        sys.exit(1)
+
+    verb = sys.argv[1]
+    path = sys.argv[2]
+    if len(sys.argv) >= 4:
+        manifest = sys.argv[3]
+    else:
+        manifest = os.path.join(path, DEFAULT_MANIFEST_NAME)
+
+    if verb == "process":
+        sys.exit(do_process(path, manifest))
+
+    if verb == "restore":
+        sys.exit(do_restore(path, manifest))
+
+    usage()
+    sys.exit(1)

user_settings.sample.py

@@ -15,6 +15,9 @@ user_settings = {
     #vkd3d debug logging
     "VKD3D_DEBUG": "warn",
 
+    #vkd3d-shader debug logging
+    "VKD3D_SHADER_DEBUG": "fixme",
+
     #wine-mono debug logging (Wine's .NET replacement)
     "WINE_MONO_TRACE": "E:System.NotImplementedException",
     #"MONO_LOG_LEVEL": "info",

vkd3d-proton

@@ -1 +1 @@
-Subproject commit 33f17cc74d341a22ddba701a68761e710e328777
+Subproject commit 1cc8afcc8e37a2b5a8f7f457fbb5406de84af293

wine

@@ -1 +1 @@
-Subproject commit bdb21c72edcccc71416c399ed432ed5eab426e26
+Subproject commit b3a69b6b2a4fec71e3f78edf9c29c0b37db72da8