Try adjusting their AI task/target priorities some, make them focus on zeds or animals first instead of the player. That way it'll give you a better chance to notice them if they are busy fighting something else in the area first.
Here is the bandit template, I'm pretty sure there is some stuff that can be removed but since its the first time I use them I'm pretty careful. I mess a lot with zombies.... but bandit its new for me.
<entity_class name="BanditTemplate">
<property name="Faction" value="banditronin"/>
<property name="Mesh" value="Player/Male/player_maleRagdoll"/>
<property name="NPCID" value="banditronin"/>
<!--<property name="AIPackages" value="Bandit Basic, Bandit Melee"/>-->
<!-- ITEMS -->
<property name="ItemsOnEnterGame.GameModeSurvivalSP" value="clubSpiked"/>
<property name="ItemsOnEnterGame.GameModeSurvivalMP" value="clubSpiked"/>
<!-- Avatar -->
<property name="Archetype" value="BanditMale"/>
<property name="Class" value="EntityBandit"/>
<property name="AvatarController" value="AvatarUMAController"/>
<property name="PhysicsBody" value="Player"/>
<property name="ModelType" value="NpcUMA"/>
<property name="EntityType" value="Zombie"/>
<property name="Prefab" value="NPC"/>
<property name="Parent" value="Enemies"/>
<property name="HasRagdoll" value="true"/>
<property name="IsEnemyEntity" value="true"/>
<property name="SurfaceCategory" value="organic"/>
<property name="ParticleOnDeath" value="blood_death"/>
<!-- AI -->
<property name="AITask-1" value="RangedAttackTarget2" param1="0" param2="5"/>
<property name="AITask-2" value="ApproachAndAttackTarget" param1="EntityPlayer"/>
<property name="AITask-3" value="ApproachAndAttackTarget" param1="EntityZombie"/>
<property name="AITask-4" value="ApproachAndAttackTarget" param1="EntityZombieCop"/>
<property name="AITask-5" value="ApproachAndAttackTarget" param1="EntityZombieDog"/>
<property name="AITask-6" value="ApproachAndAttackTarget" param1="EntityZombieCrawl"/>
<property name="AITask-7" value="ApproachAndAttackTarget" param1="EntityHornet"/>
<property name="AITask-8" value="ApproachAndAttackTarget" param1="EntityAnimalBear"/>
<property name="AITask-9" value="ApproachAndAttackTarget" param1="EntityAnimalStag"/>
<property name="AITask-10" value="ApproachAndAttackTarget" param1="EntityAnimalRabbit"/>
<property name="AITask-11" value="ApproachAndAttackTarget" param1="EntitySurvivor"/>
<property name="AITask-12" value="BreakDoor" />
<property name="AITask-13" value="BreakBlock" />
<property name="AITask-14" value="Swim"/>
<property name="AITask-15" value="ApproachSpot"/>
<property name="AITask-16" value="Wander"/>
<property name="AITarget-1" value="SetAsTargetIfHurt" param1="EntityPlayer"/>
<property name="AITarget-2" value="SetAsTargetIfHurt" param1="EntityZombieCop"/>
<property name="AITarget-3" value="SetAsTargetIfHurt" param1="EntityZombieDog"/>
<property name="AITarget-4" value="SetAsTargetIfHurt" param1="EntityZombie"/>
<property name="AITarget-5" value="SetAsTargetIfHurt" param1="EntityZombieCrawl"/>
<property name="AITarget-6" value="SetAsTargetIfHurt" param1="EntityHornet"/>
<property name="AITarget-7" value="SetAsTargetIfHurt" param1="EntityAnimalBear"/>
<property name="AITarget-8" value="SetAsTargetIfHurt" param1="EntityAnimalStag"/>
<property name="AITarget-9" value="SetAsTargetIfHurt" param1="EntityAnimalRabbit"/>
<property name="AITarget-10" value="SetAsTargetIfHurt" param1="EntitySurvivor"/>
<property name="AITarget-11" value="SetNearestEntityAsTarget" param1="EntityPlayer"/>
<property name="AITarget-12" value="SetNearestEntityAsTarget" param1="EntityZombie"/>
<property name="AITarget-13" value="SetNearestEntityAsTarget" param1="EntityZombieCop"/>
<property name="AITarget-14" value="SetNearestEntityAsTarget" param1="EntityZombieDog"/>
<property name="AITarget-15" value="SetNearestEntityAsTarget" param1="EntityZombieCrawl"/>
<property name="AITarget-16" value="SetNearestEntityAsTarget" param1="EntityHornet"/>
<property name="AITarget-17" value="SetNearestEntityAsTarget" param1="EntityAnimalBear"/>
<property name="AITarget-18" value="SetNearestEntityAsTarget" param1="EntityAnimalStag"/>
<property name="AITarget-19" value="SetNearestEntityAsTarget" param1="EntityAnimalRabbit"/>
<property name="AITarget-20" value="SetNearestEntityAsTarget" param1="EntitySurvivor"/>
<!-- Movement -->
<property name="WanderSpeed" value="0.5"/>
<property name="ApproachSpeed" value="1"/>
<property name="NightWanderSpeed" value="0.5"/>
<property name="NightApproachSpeed" value="1"/>
<property name="PanicSpeed" value="1.15"/>
<property name="CanClimbLadders" value="true"/>
<property name="SwimOffset" value="1.0"/>
<!-- Sounds -->
<property name="SoundRandomTime" value="200.0"/>
<property name="SoundRandom" value="malehateroam"/>
<property name="SoundAlert" value="malehatealert"/>
<property name="SoundSense" value="malehatesense"/>
<property name="SoundHurt" value="malehatepain"/>
<property name="SoundDeath" value="malehatedeath"/>
<property name="SoundAttack" value="malehateattack"/>
<!-- Stats -->
<property name="MaxHealth" value="150"/>
<property name="ExperienceGain" value="1275"/>
<property name="CorpseBlock" value="GoreBlock1Prefab"/>
<property name="CorpseBlockChance" value="1"/>
<property name="MaxViewAngle" value="270"/>
<property name="TimeStayAfterDeath" value="5"/>
<property name="DeadBodyHitPoints" value="20"/>
<property name="HasDeathAnim" value="true"/>
<property name="Weight" value="70"/>
<property name="SightRange" value="50"/>
<property name="SightScale" value="-2,50"/>
</entity_class>