How two avoid cmake repeat build for two similar libraries

I am planning to add unittests to a project whose product is a shared library. The original library limits the public symbols:

function(export_signatures TARGET_NAME)
    target_link_options(${TARGET_NAME} PRIVATE
    -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.linux.list -Wl,--no-undefined)
endfunction()

This will give undefined reference if I use internal symbols in my test. So I figured I could build a version of the library to avoid repeating build steps.
So I created a loop:

set(MY_TARGETS mylib)
if (ENABLE_TEST_BUILD)
    list(APPEND MY_TARGETS mylib-test)
endif()

foreach(TARGET_NAME ${MY_TARGETS})
    add_library(${TARGET_NAME} SHARED ${SOURCES})
    if (TARGET_NAME STREQUAL "mylib") #skip test library
        export_signatures(${TARGET_NAME})
    endif()
    install(TARGETS ${TARGET_NAME} DESTINATION "${CMAKE_INSTALL_PREFIX}/lib")
endforeach()

And now I can see that sources are visited twice, and my build time is increased by 50%.

[ 99%] Building CXX object path/CMakeFiles/mylib-test.dir/util.cpp.o
[ 100%] Building CXX object path/CMakeFiles/mylib.dir/util.cpp.o

Is there a way to avoid that? More generic question: Is this a sane design anyway? What'd you suggest?

Solution

Thanks to the comments, I solved the issue like this:

file(GLOB SOURCES *.cpp)
# Remove shared library specific file(s)
list(REMOVE_ITEM ${SOURCES} dllmain.cpp)

# Build only Object files
add_library(objects OBJECT ${SOURCES})

add_library(target1 SHARED dllmain.cpp $<TARGET_OBJECTS:objects>)
# target1 specific setting here ...

add_library(target2 SHARED dllmain.cpp $<TARGET_OBJECTS:objects>)
# target2 specific setting here ...